New Release v0.6.0 - #minor

.github/pull_request_template.md

@@ -1,2 +1,2 @@
 
-- [ ] Model changes documented in the [business entity model](https://ltwiki.adr.admin.ch:8443/display/PB/Business+Entity+Model)
+- [ ] Model changes documented in the [business entity model](https://swissgeoplatform.atlassian.net/wiki/spaces/PB/pages/16155637/Business+Entity+Model)

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-django = "~=5.0.13"
+django = "~=5.0.14"
 django-ninja = "~=1.1"
 psycopg2-binary = "~=2.9.5"
 django-environ = "~=0.11"
@@ -17,6 +17,8 @@ nanoid = "~=2.0.0"
 whitenoise = "~=6.8.2"
 pystac-client = "~=0.8.5"
 ecs-logging = "*"
+beautifulsoup4 = "*"
+requests = "*"
 
 [dev-packages]
 yapf = "*"
@@ -40,6 +42,7 @@ bandit = "*"
 pytest-xdist = "*"
 types-nanoid = "*"
 pytest-cov = "*"
+types-requests = "*"
 
 [requires]
 python_version = "3.12"

app/bod/management/commands/bod_sync.py

@@ -5,6 +5,7 @@
 
 from bod.models import BodContactOrganisation
 from bod.models import BodDataset
+from bod.models import BodGeocatPublish
 from bod.models import BodTranslations
 from distributions.models import Attribution
 from distributions.models import Dataset
@@ -271,6 +272,36 @@ def import_datasets(self) -> None:
                 )
                 continue
 
+            if legacy_id in (1485, 1486):
+                # Skip the following datasets as they are hidden in public BGDI services:
+                # - ch.swisstopo.konsultation-lk10-flurnamen (1485)
+                # - ch.swisstopo.konsultation-lk10-siedlungsnamen (1486)
+                # They were used for name verifications of the cantons in the context of the KONAM
+                # project and are to be deleted.
+                # Layer ch.swisstopo.landeskarte-farbe-10 (952) has the same
+                # geocat_id ("cb0f8401-c49a-4bdf-aff6-40a7015ba43a").
+                self.print(f"skipping dataset '{bod_dataset.id_dataset}'")
+                continue
+
+            # Get meta information title and description
+            bod_meta = BodGeocatPublish.objects.filter(fk_id_dataset=bod_dataset.id_dataset).first()
+            if not bod_meta:
+                bod_meta = BodGeocatPublish()
+
+            if not bod_meta.bezeichnung_de:
+                bod_meta.bezeichnung_de = "#Missing"
+            if not bod_meta.bezeichnung_fr:
+                bod_meta.bezeichnung_fr = "#Missing"
+            if not bod_meta.abstract_de:
+                bod_meta.abstract_de = "#Missing"
+            if not bod_meta.abstract_fr:
+                bod_meta.abstract_fr = "#Missing"
+            # If english is missing, geocat defaults to german translation
+            if not bod_meta.bezeichnung_en:
+                bod_meta.bezeichnung_en = bod_meta.bezeichnung_de
+            if not bod_meta.abstract_en:
+                bod_meta.abstract_en = bod_meta.abstract_de
+
             # Get or create dataset
             is_new_model = False
             dataset = Dataset.objects.filter(
@@ -281,13 +312,24 @@ def import_datasets(self) -> None:
                 dataset = Dataset.objects.create(
                     provider=attribution.provider,
                     attribution=attribution,
+                    geocat_id=bod_dataset.fk_geocat,
+                    title_de=bod_meta.bezeichnung_de,
+                    title_fr=bod_meta.bezeichnung_fr,
+                    title_en=bod_meta.bezeichnung_en,
+                    title_it=bod_meta.bezeichnung_it,
+                    title_rm=bod_meta.bezeichnung_rm,
+                    description_de=bod_meta.abstract_de,
+                    description_fr=bod_meta.abstract_fr,
+                    description_en=bod_meta.abstract_en,
+                    description_it=bod_meta.abstract_it,
+                    description_rm=bod_meta.abstract_rm,
                     _legacy_id=legacy_id,
                     dataset_id='undefined'
                 )
                 self.increment_counter('dataset', 'added')
                 self.print(f"Added dataset '{bod_dataset.id_dataset}'")
 
-            self.update_dataset(dataset, bod_dataset, is_new_model)
+            self.update_dataset(dataset, bod_dataset, bod_meta, is_new_model)
 
             dataset.save()
 
@@ -298,18 +340,39 @@ def import_datasets(self) -> None:
             model_class = model.split('.')[-1].lower()
             self.increment_counter(model_class, 'removed', count)
 
-    def update_dataset(self, dataset: Dataset, bod_dataset: BodDataset, is_new_model: bool) -> None:
+    def update_dataset(
+        self,
+        dataset: Dataset,
+        bod_dataset: BodDataset,
+        bod_meta: BodGeocatPublish,
+        is_new_model: bool
+    ) -> None:
         """ Update the attributes of a dataset. """
 
         any_changed = False
-        for dataset_attribute, bod_dataset_attribute in (('dataset_id', 'id_dataset'),):
+        for dataset_attribute, bod_dataset_attribute in (('dataset_id', 'id_dataset'),
+            ('geocat_id', 'fk_geocat'),):
             changed = self.update_model(
                 dataset,
                 dataset_attribute,
                 getattr(bod_dataset, bod_dataset_attribute),
                 is_new_model
             )
             any_changed = any_changed or changed
+        for dataset_attribute, bod_dataset_attribute in (('title_de', 'bezeichnung_de'),
+            ('title_fr', 'bezeichnung_fr'),
+            ('title_en', 'bezeichnung_en'),
+            ('title_it', 'bezeichnung_it'),
+            ('title_rm', 'bezeichnung_rm'),
+            ('description_de', 'abstract_de'),
+            ('description_fr', 'abstract_fr'),
+            ('description_en', 'abstract_en'),
+            ('description_it', 'abstract_it'),
+            ('description_rm', 'abstract_rm'),):
+            changed = self.update_model(
+                dataset, dataset_attribute, getattr(bod_meta, bod_dataset_attribute), is_new_model
+            )
+            any_changed = any_changed or changed
         if any_changed and not is_new_model:
             self.increment_counter('dataset', 'updated')
 

app/bod/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.12 on 2025-03-11 11:57
+# Generated by Django 5.0.14 on 2025-04-10 07:12
 
 from django.conf import settings
 from django.db import migrations
@@ -38,6 +38,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(primary_key=True, serialize=False, unique=True)),
                 ('id_dataset', models.TextField()),
+                ('fk_geocat', models.TextField()),
                 ('fk_contactorganisation_id', models.IntegerField(blank=True, null=True)),
                 ('staging', models.TextField(blank=True, null=True)),
             ],
@@ -46,6 +47,27 @@ class Migration(migrations.Migration):
                 'managed': settings.TESTING,
             },
         ),
+        migrations.CreateModel(
+            name='BodGeocatPublish',
+            fields=[
+                ('bgdi_id', models.AutoField(primary_key=True, serialize=False, unique=True)),
+                ('fk_id_dataset', models.TextField()),
+                ('bezeichnung_de', models.CharField(blank=True, max_length=5000, null=True)),
+                ('bezeichnung_fr', models.CharField(blank=True, max_length=5000, null=True)),
+                ('bezeichnung_it', models.CharField(blank=True, max_length=5000, null=True)),
+                ('bezeichnung_rm', models.CharField(blank=True, max_length=5000, null=True)),
+                ('bezeichnung_en', models.CharField(blank=True, max_length=5000, null=True)),
+                ('abstract_de', models.CharField(blank=True, max_length=5000, null=True)),
+                ('abstract_fr', models.CharField(blank=True, max_length=5000, null=True)),
+                ('abstract_it', models.CharField(blank=True, max_length=5000, null=True)),
+                ('abstract_rm', models.CharField(blank=True, max_length=5000, null=True)),
+                ('abstract_en', models.CharField(blank=True, max_length=5000, null=True)),
+            ],
+            options={
+                'db_table': 'geocat_publish',
+                'managed': settings.TESTING,
+            },
+        ),
         migrations.CreateModel(
             name='BodTranslations',
             fields=[

app/bod/models.py

@@ -27,6 +27,7 @@ class Meta:
 class BodDataset(models.Model):
     id = models.AutoField(unique=True, primary_key=True)
     id_dataset = models.TextField()
+    fk_geocat = models.TextField()
     fk_contactorganisation_id = models.IntegerField(blank=True, null=True)
     staging = models.TextField(blank=True, null=True)
 
@@ -47,3 +48,24 @@ class BodTranslations(models.Model):
     class Meta:
         managed = settings.TESTING
         db_table = 'translations'
+
+
+class BodGeocatPublish(models.Model):
+    bgdi_id = models.AutoField(unique=True, primary_key=True)
+    fk_id_dataset = models.TextField()
+
+    bezeichnung_de = models.CharField(max_length=5000, blank=True, null=True)
+    bezeichnung_fr = models.CharField(max_length=5000, blank=True, null=True)
+    bezeichnung_it = models.CharField(max_length=5000, blank=True, null=True)
+    bezeichnung_rm = models.CharField(max_length=5000, blank=True, null=True)
+    bezeichnung_en = models.CharField(max_length=5000, blank=True, null=True)
+
+    abstract_de = models.CharField(max_length=5000, blank=True, null=True)
+    abstract_fr = models.CharField(max_length=5000, blank=True, null=True)
+    abstract_it = models.CharField(max_length=5000, blank=True, null=True)
+    abstract_rm = models.CharField(max_length=5000, blank=True, null=True)
+    abstract_en = models.CharField(max_length=5000, blank=True, null=True)
+
+    class Meta:
+        managed = settings.TESTING
+        db_table = 'geocat_publish'

app/config/logging-cfg-local.yaml

@@ -4,7 +4,7 @@ disable_existing_loggers: False
 root:
   handlers:
     - console
-  level: DEBUG
+  level: INFO
   propagate: True
 
 # configure loggers per app

app/distributions/admin.py

@@ -1,4 +1,8 @@
+from typing import Any
+
 from django.contrib import admin
+from django.http import HttpRequest
+from django.utils.html import format_html
 
 from .models import Attribution
 from .models import Dataset
@@ -17,13 +21,34 @@ class AttributionAdmin(admin.ModelAdmin):  # type:ignore[type-arg]
 class DatasetAdmin(admin.ModelAdmin):  # type:ignore[type-arg]
     '''Admin View for Dataset'''
 
-    list_display = ('dataset_id', 'provider')
+    list_display = ('dataset_id', 'title_en', 'provider')
     list_filter = (('provider', admin.RelatedOnlyFieldListFilter),)
 
+    def get_form(
+        self,
+        request: HttpRequest,
+        obj: Dataset | None = None,
+        change: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        form = super().get_form(request, obj=obj, change=change, **kwargs)
+        if obj is not None:
+            self._add_geocat_url_help_text(form, obj.geocat_id)
+        return form
+
+    def _add_geocat_url_help_text(self, form: Any, geocat_id: str) -> None:
+        form.base_fields["geocat_id"].help_text = format_html(
+            "<a href='{url}'>{url}</a>",
+            url=f"https://www.geocat.ch/datahub/dataset/{geocat_id}",
+        )
+
 
 @admin.register(PackageDistribution)
 class PackageDistributionAdmin(admin.ModelAdmin):  # type:ignore[type-arg]
     '''Admin View for Package Distribution'''
 
     list_display = ('package_distribution_id', 'managed_by_stac', 'dataset')
-    list_filter = ('managed_by_stac',)
+    list_filter = (
+        'managed_by_stac',
+        ('dataset__provider', admin.RelatedOnlyFieldListFilter),
+    )

app/distributions/api.py

@@ -45,12 +45,28 @@ def attribution_to_response(model: Attribution, lang: LanguageCode) -> Attributi
     return response
 
 
-def dataset_to_response(model: Dataset) -> DatasetSchema:
+def dataset_to_response(model: Dataset, lang: LanguageCode) -> DatasetSchema:
     """
-    Maps the given model to the corresponding schema.
+    Transforms the given model using the given language into a response object.
     """
     response = DatasetSchema(
         id=model.dataset_id,
+        title=get_translation(model, "title", lang),
+        title_translations=TranslationsSchema(
+            de=model.title_de,
+            fr=model.title_fr,
+            en=model.title_en,
+            it=model.title_it,
+            rm=model.title_rm,
+        ),
+        description=get_translation(model, "description", lang),
+        description_translations=TranslationsSchema(
+            de=model.description_de,
+            fr=model.description_fr,
+            en=model.description_en,
+            it=model.description_it,
+            rm=model.description_rm,
+        ),
         created=model.created,
         updated=model.updated,
         provider_id=model.provider.provider_id,
@@ -148,12 +164,15 @@ def attributions(request: HttpRequest,
     exclude_none=True,
     auth=PermissionAuth('distributions.view_dataset')
 )
-def dataset(request: HttpRequest, dataset_id: str) -> DatasetSchema:
+def dataset(
+    request: HttpRequest, dataset_id: str, lang: LanguageCode | None = None
+) -> DatasetSchema:
     """
     Get the dataset with the given ID.
     """
     model = get_object_or_404(Dataset, dataset_id=dataset_id)
-    response = dataset_to_response(model)
+    lang_to_use = get_language(lang, request.headers)
+    response = dataset_to_response(model, lang_to_use)
     return response
 
 
@@ -163,14 +182,15 @@ def dataset(request: HttpRequest, dataset_id: str) -> DatasetSchema:
     exclude_none=True,
     auth=PermissionAuth('distributions.view_dataset')
 )
-def datasets(request: HttpRequest) -> dict[str, list[DatasetSchema]]:
+def datasets(request: HttpRequest,
+             lang: LanguageCode | None = None) -> dict[str, list[DatasetSchema]]:
     """
     Get all datasets.
 
     For more details on how individual datasets are returned, see the
     corresponding endpoint for a specific attribution.
     """
     models = Dataset.objects.order_by("dataset_id").all()
-
-    responses = [dataset_to_response(model) for model in models]
+    lang_to_use = get_language(lang, request.headers)
+    responses = [dataset_to_response(model, lang_to_use) for model in models]
     return {"items": responses}