[#2483] Update cases listview to show the combined data from multiple endpoints

src/open_inwoner/cms/cases/views/cases.py

@@ -1,22 +1,42 @@
+import concurrent.futures
+import logging
+from dataclasses import dataclass
+
 from django.urls import reverse
 from django.utils.functional import cached_property
 from django.utils.translation import gettext_lazy as _
 from django.views.generic import TemplateView
 
 from view_breadcrumbs import BaseBreadcrumbMixin
+from zgw_consumers.concurrent import parallel
 
 from open_inwoner.htmx.mixins import RequiresHtmxMixin
+from open_inwoner.openzaak.api_models import Zaak
 from open_inwoner.openzaak.cases import preprocess_data
-from open_inwoner.openzaak.clients import build_zaken_client
 from open_inwoner.openzaak.formapi import fetch_open_submissions
-from open_inwoner.openzaak.models import OpenZaakConfig
+from open_inwoner.openzaak.models import OpenZaakConfig, ZGWApiGroupConfig
 from open_inwoner.openzaak.types import UniformCase
 from open_inwoner.openzaak.utils import get_user_fetch_parameters
 from open_inwoner.utils.mixins import PaginationMixin
 from open_inwoner.utils.views import CommonPageMixin
 
 from .mixins import CaseAccessMixin, CaseLogMixin, OuterCaseAccessMixin
 
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ZaakWithApiGroup(UniformCase):
+    zaak: Zaak
+    api_group: ZGWApiGroupConfig
+
+    @property
+    def identifier(self):
+        return self.zaak.url
+
+    def process_data(self) -> dict:
+        return {**self.zaak.process_data(), "api_group": self.api_group}
+
 
 class OuterCaseListView(
     OuterCaseAccessMixin, CommonPageMixin, BaseBreadcrumbMixin, TemplateView
@@ -55,17 +75,36 @@ class InnerCaseListView(
     def page_title(self):
         return _("Mijn aanvragen")
 
-    def get_cases(self):
-        client = build_zaken_client()
-
-        if client is None:
-            return []
-
-        raw_cases = client.fetch_cases(**get_user_fetch_parameters(self.request))
-
-        preprocessed_cases = preprocess_data(raw_cases)
+    def get_cases_for_api_group(self, group: ZGWApiGroupConfig):
+        raw_cases = group.zaken_client.fetch_cases(
+            **get_user_fetch_parameters(self.request)
+        )
+        preprocessed_cases = preprocess_data(raw_cases, group)
         return preprocessed_cases
 
+    def get_cases(self) -> list[ZaakWithApiGroup]:
+        all_api_groups = list(ZGWApiGroupConfig.objects.all())
+        with parallel() as executor:
+            futures = [
+                executor.submit(self.get_cases_for_api_group, group)
+                for group in all_api_groups
+            ]
+
+            cases = []
+            for task in concurrent.futures.as_completed(futures):
+                try:
+                    group_for_task = all_api_groups[futures.index(task)]
+                    for row in task.result():
+                        cases.append(
+                            ZaakWithApiGroup(zaak=row, api_group=group_for_task)
+                        )
+                except BaseException:
+                    logger.exception("Error fetching and pre-processing cases")
+
+        # Ensure stable sorting for pagination and testing purposes
+        cases.sort(key=lambda c: all_api_groups.index(c.api_group))
+        return cases
+
     def get_submissions(self):
         subs = fetch_open_submissions(self.request.user.bsn)
         subs.sort(key=lambda sub: sub.datum_laatste_wijziging, reverse=True)

src/open_inwoner/openzaak/cases.py

@@ -6,27 +6,21 @@
 from zgw_consumers.concurrent import parallel
 
 from .api_models import Zaak
-from .clients import (
-    CatalogiClient,
-    ZakenClient,
-    build_catalogi_client,
-    build_zaken_client,
-)
-from .models import ZaakTypeConfig, ZaakTypeStatusTypeConfig
+from .clients import CatalogiClient, ZakenClient
+from .models import ZaakTypeConfig, ZaakTypeStatusTypeConfig, ZGWApiGroupConfig
 from .utils import is_zaak_visible
 
 logger = logging.getLogger(__name__)
 
 
-def resolve_zaak_type(case: Zaak, client: CatalogiClient | None = None) -> None:
+def resolve_zaak_type(case: Zaak, client: CatalogiClient) -> None:
     """
     Resolve `case.zaaktype` (`str`) to a `ZaakType(ZGWModel)` object
 
     Note: the result of `fetch_single_case_type` is cached, hence a request
           is only made for new case type urls
     """
     case_type_url = case.zaaktype
-    client = client or build_catalogi_client()
     if client:
         case_type = client.fetch_single_case_type(case_type_url)
         case.zaaktype = case_type
@@ -36,7 +30,6 @@ def resolve_status(case: Zaak, client: ZakenClient | None = None) -> None:
     """
     Resolve `case.status` (`str`) to a `Status(ZGWModel)` object
     """
-    client = client or build_zaken_client()
     if client:
         case.status = client.fetch_single_status(case.status)
 
@@ -46,7 +39,6 @@ def resolve_status_type(case: Zaak, client: CatalogiClient | None = None) -> Non
     Resolve `case.status.statustype` (`str`) to a `StatusType(ZGWModel)` object
     """
     statustype_url = case.status.statustype
-    client = client or build_catalogi_client()
     if client:
         case.status.statustype = client.fetch_single_status_type(statustype_url)
 
@@ -55,16 +47,14 @@ def resolve_resultaat(case: Zaak, client: ZakenClient | None = None) -> None:
     """
     Resolve `case.resultaat` (`str`) to a `Resultaat(ZGWModel)` object
     """
-    client = client or build_zaken_client()
-    if client and case.resultaat:
+    if case.resultaat:
         case.resultaat = client.fetch_single_result(case.resultaat)
 
 
 def resolve_resultaat_type(case: Zaak, client: CatalogiClient | None = None) -> None:
     """
     Resolve `case.resultaat.resultaattype` (`str`) to a `ResultaatType(ZGWModel)` object
     """
-    client = client or build_catalogi_client()
     if client and case.resultaat:
         case.resultaat.resultaattype = client.fetch_single_resultaat_type(
             case.resultaat.resultaattype
@@ -102,30 +92,27 @@ def add_status_type_config(case: Zaak) -> None:
         pass
 
 
-def preprocess_data(cases: list[Zaak]) -> list[Zaak]:
+def preprocess_data(cases: list[Zaak], group: ZGWApiGroupConfig) -> list[Zaak]:
     """
     Resolve zaaktype and statustype, add status type config, filter for visibility
 
     Note: we need to iterate twice over `cases` because the `zaak_type` must be
           resolved to a `ZaakType` object before we can filter by visibility
     """
-    zaken_client = build_zaken_client()
-    catalogi_client = build_catalogi_client()
 
     def preprocess_case(case: Zaak) -> None:
-        resolve_status(case, client=zaken_client)
-        resolve_status_type(case, client=catalogi_client)
-        resolve_resultaat(case, client=zaken_client)
-        resolve_resultaat_type(case, client=catalogi_client)
+        resolve_status(case, client=group.zaken_client)
+        resolve_status_type(case, client=group.catalogi_client)
+        resolve_resultaat(case, client=group.zaken_client)
+        resolve_resultaat_type(case, client=group.catalogi_client)
         add_zaak_type_config(case)
         add_status_type_config(case)
 
-    # TODO error handling if these are none?
     # use contextmanager to ensure the `requests.Session` is reused
-    with zaken_client, catalogi_client:
+    with group.catalogi_client, group.zaken_client:
         with parallel(max_workers=settings.CASE_LIST_NUM_THREADS) as executor:
             futures = [
-                executor.submit(resolve_zaak_type, case, client=catalogi_client)
+                executor.submit(resolve_zaak_type, case, client=group.catalogi_client)
                 for case in cases
             ]
             concurrent.futures.wait(futures)

src/open_inwoner/openzaak/models.py

@@ -69,6 +69,16 @@ class ZGWApiGroupConfig(models.Model):
         null=False,
         blank=False,
     )
+
+    def _build_client_from_attr(self, attr: str):
+        from .clients import build_zgw_client_from_service
+
+        return build_zgw_client_from_service(getattr(self, attr))
+
+    @property
+    def zaken_client(self):
+        return self._build_client_from_attr("zrc_service")
+
     drc_service = models.ForeignKey(
         "zgw_consumers.Service",
         verbose_name=_("Documenten API"),
@@ -78,6 +88,11 @@ class ZGWApiGroupConfig(models.Model):
         null=False,
         blank=False,
     )
+
+    @property
+    def documenten_client(self):
+        return self._build_client_from_attr("drc_service")
+
     ztc_service = models.ForeignKey(
         "zgw_consumers.Service",
         verbose_name=_("Catalogi API"),
@@ -87,6 +102,11 @@ class ZGWApiGroupConfig(models.Model):
         null=False,
         blank=False,
     )
+
+    @property
+    def catalogi_client(self):
+        return self._build_client_from_attr("ztc_service")
+
     form_service = models.OneToOneField(
         "zgw_consumers.Service",
         verbose_name=_("Form API"),
@@ -97,6 +117,11 @@ class ZGWApiGroupConfig(models.Model):
         blank=True,
     )
 
+    @property
+    def forms_client(self):
+        if self.form_service:
+            return self._build_client_from_attr("form_service")
+
     class Meta:
         verbose_name = _("ZGW API set")
         verbose_name_plural = _("ZGW API sets")