Merge pull request #77 from geoadmin/feat-PB-1489-stagings

PB-1489: Skip non-prod datasets in bod_sync

Author: msom

app/bod/management/commands/bod_sync.py

@@ -79,9 +79,7 @@ def import_providers(self) -> None:
         processed = set()
 
         for organization in BodContactOrganisation.objects.all():
-            if organization.attribution is not None and len(
-                organization.attribution.split('.')
-            ) != 2:
+            if not organization.attribution or len(organization.attribution.split('.')) != 2:
                 # Skip entries that are not a provider.
                 # BodContactOrganisation (table 'contactorganisation') contains providers and
                 # attributions. Providers are of the format "ch.<short_name>" and only include a
@@ -99,7 +97,7 @@ def import_providers(self) -> None:
                 is_new_model = True
                 provider = Provider.objects.create(
                     _legacy_id=legacy_id,
-                    provider_id=organization.attribution,  # type:ignore[misc]
+                    provider_id=organization.attribution,
                     acronym_de="undefined",
                     acronym_fr="undefined",
                     acronym_en="undefined",
@@ -171,8 +169,8 @@ def import_attribution(self) -> None:
             processed.add(legacy_id)
 
             # Get related provider
-            provider: Provider | None
-            if organization.attribution is not None:
+            provider = None
+            if organization.attribution:
                 provider_of_attribution = ".".join(organization.attribution.split(".", 2)[:2])
                 provider = Provider.objects.filter(provider_id=provider_of_attribution).first()
             if not provider:
@@ -256,7 +254,7 @@ def import_datasets(self) -> None:
         """
 
         processed = set()
-        for bod_dataset in BodDataset.objects.all():
+        for bod_dataset in BodDataset.objects.filter(staging='prod').all():
             # Keep track of processed BOD datasets for orphan removal
             legacy_id = bod_dataset.id
             processed.add(legacy_id)
@@ -273,24 +271,15 @@ def import_datasets(self) -> None:
                 )
                 continue
 
-            # Get related provider
-            provider = Provider.objects.filter(id=attribution.provider_id).first()
-            if not provider:
-                # Skip as no matching provider
-                self.print(
-                    f"skipping dataset '{bod_dataset.id_dataset}' as no matching provider was found"
-                )
-                continue
-
             # Get or create dataset
             is_new_model = False
             dataset = Dataset.objects.filter(
-                provider=provider, attribution=attribution, _legacy_id=legacy_id
+                provider=attribution.provider, attribution=attribution, _legacy_id=legacy_id
             ).first()
             if not dataset:
                 is_new_model = True
                 dataset = Dataset.objects.create(
-                    provider=provider,
+                    provider=attribution.provider,
                     attribution=attribution,
                     _legacy_id=legacy_id,
                     dataset_id='undefined'

app/bod/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.9 on 2024-10-03 13:55
+# Generated by Django 5.0.12 on 2025-03-11 11:57
 
 from django.conf import settings
 from django.db import migrations
@@ -39,6 +39,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(primary_key=True, serialize=False, unique=True)),
                 ('id_dataset', models.TextField()),
                 ('fk_contactorganisation_id', models.IntegerField(blank=True, null=True)),
+                ('staging', models.TextField(blank=True, null=True)),
             ],
             options={
                 'db_table': 'dataset',

app/bod/models.py

@@ -28,6 +28,7 @@ class BodDataset(models.Model):
     id = models.AutoField(unique=True, primary_key=True)
     id_dataset = models.TextField()
     fk_contactorganisation_id = models.IntegerField(blank=True, null=True)
+    staging = models.TextField(blank=True, null=True)
 
     class Meta:
         managed = settings.TESTING

app/tests/bod/test_bod_sync_command.py

@@ -41,7 +41,8 @@ def fixture_bod_dataset(bod_contact_organisation):
     yield BodDataset.objects.create(
         id=170,
         id_dataset="ch.bafu.auen-vegetationskarten",
-        fk_contactorganisation_id=bod_contact_organisation.pk_contactorganisation_id
+        fk_contactorganisation_id=bod_contact_organisation.pk_contactorganisation_id,
+        staging="prod"
     )
 
 
@@ -131,17 +132,22 @@ def test_command_imports_providers(bod_dataset):
     assert provider.acronym_rm == "UFAM"
 
 
-def test_command_imports_attributions(bod_contact_organisation, bod_dataset):
-    out = StringIO()
-    call_command(
-        "bod_sync", providers=False, attributions=True, datasets=False, verbosity=2, stdout=out
-    )
-    assert "skipping attribution 'ch.bafu' as no matching provider was found" in out.getvalue()
-    assert 'nothing to be done, already in sync' in out.getvalue()
-    assert Provider.objects.count() == 0
-    assert Attribution.objects.count() == 0
-    assert Dataset.objects.count() == 0
+def test_command_skips_invalid_providers(bod_contact_organisation, bod_dataset):
+    for attribution in ("somethingelse", "", None):
+        bod_contact_organisation.attribution = attribution
+        bod_contact_organisation.save()
+
+        out = StringIO()
+        call_command(
+            "bod_sync", providers=True, attributions=False, datasets=False, verbosity=2, stdout=out
+        )
+        assert "nothing to be done" in out.getvalue()
+        assert Provider.objects.count() == 0
+        assert Attribution.objects.count() == 0
+        assert Dataset.objects.count() == 0
 
+
+def test_command_imports_attributions(bod_contact_organisation, bod_dataset):
     provider = Provider.objects.create(
         provider_id="ch.bafu",
         name_de="XXX",
@@ -152,6 +158,8 @@ def test_command_imports_attributions(bod_contact_organisation, bod_dataset):
         acronym_en="XXX",
         _legacy_id=bod_contact_organisation.pk_contactorganisation_id
     )
+
+    out = StringIO()
     call_command(
         "bod_sync", providers=False, attributions=True, datasets=False, verbosity=2, stdout=out
     )
@@ -175,20 +183,24 @@ def test_command_imports_attributions(bod_contact_organisation, bod_dataset):
     assert attribution.description_rm == "UFAM"
 
 
-def test_command_imports_datasets(bod_contact_organisation, bod_dataset):
-    out = StringIO()
-    call_command(
-        "bod_sync", providers=False, attributions=False, datasets=True, verbosity=2, stdout=out
-    )
-    assert (
-        "skipping dataset 'ch.bafu.auen-vegetationskarten' " +
-        "as no matching attribution was found"
-    ) in out.getvalue()
-    assert 'nothing to be done, already in sync' in out.getvalue()
-    assert Provider.objects.count() == 0
-    assert Attribution.objects.count() == 0
-    assert Dataset.objects.count() == 0
+def test_command_skips_invalid_attributions(bod_contact_organisation, bod_dataset):
+    for attribution in (bod_contact_organisation.attribution, "", None):
+        bod_contact_organisation.attribution = attribution
+        bod_contact_organisation.save()
+
+        out = StringIO()
+        call_command(
+            "bod_sync", providers=False, attributions=True, datasets=False, verbosity=2, stdout=out
+        )
+        assert (f"skipping attribution '{attribution}' as no matching provider was found"
+               ) in out.getvalue()
+        assert 'nothing to be done, already in sync' in out.getvalue()
+        assert Provider.objects.count() == 0
+        assert Attribution.objects.count() == 0
+        assert Dataset.objects.count() == 0
+
 
+def test_command_imports_datasets(bod_contact_organisation, bod_dataset):
     provider = Provider.objects.create(
         provider_id="ch.bafu",
         name_de="XXX",
@@ -210,6 +222,8 @@ def test_command_imports_datasets(bod_contact_organisation, bod_dataset):
         provider=provider,
         _legacy_id=bod_contact_organisation.pk_contactorganisation_id
     )
+
+    out = StringIO()
     call_command(
         "bod_sync", providers=False, attributions=False, datasets=True, verbosity=2, stdout=out
     )
@@ -224,6 +238,34 @@ def test_command_imports_datasets(bod_contact_organisation, bod_dataset):
     assert dataset.dataset_id == "ch.bafu.auen-vegetationskarten"
 
 
+def test_command_skips_invalid_datasets(bod_dataset):
+    out = StringIO()
+    call_command(
+        "bod_sync", providers=False, attributions=False, datasets=True, verbosity=2, stdout=out
+    )
+    assert (
+        "skipping dataset 'ch.bafu.auen-vegetationskarten' " +
+        "as no matching attribution was found"
+    ) in out.getvalue()
+    assert 'nothing to be done, already in sync' in out.getvalue()
+    assert Provider.objects.count() == 0
+    assert Attribution.objects.count() == 0
+    assert Dataset.objects.count() == 0
+
+
+def test_command_skips_non_prod_datasets(bod_dataset):
+    bod_dataset.staging = 'test'
+
+    out = StringIO()
+    call_command(
+        "bod_sync", providers=False, attributions=False, datasets=True, verbosity=2, stdout=out
+    )
+    assert 'nothing to be done, already in sync' in out.getvalue()
+    assert Provider.objects.count() == 0
+    assert Attribution.objects.count() == 0
+    assert Dataset.objects.count() == 0
+
+
 def test_command_updates(bod_contact_organisation, bod_dataset):
     # Add objects that will be updated
     provider = Provider.objects.create(