Merge pull request #3410 from GeotrekCE/impr-apidae-trek-parser

💫 Various improvements for the APIDAE trek parser

docs/changelog.rst

@@ -33,6 +33,11 @@ In preparation for HD Views developments (PR #3298)
 
 - Add libvips to dependencies
 
+**Improvements**
+
+- Apidae trek parser supports geometry import from kml or kmz attachment
+- More checks on Apidae trek parser in order not to import trek without a geometry
+
 **Bug fixes**
 
 - Recreate cache folders if missing. (#3384)

geotrek/trekking/parsers.py

@@ -1,9 +1,11 @@
 from datetime import date, timedelta
 from decimal import Decimal
+import io
 import json
 from collections import defaultdict
 import re
 from tempfile import NamedTemporaryFile
+import zipfile
 
 from django.conf import settings
 from django.contrib.gis.gdal import DataSource
@@ -460,7 +462,8 @@ class ApidaeTrekParser(AttachmentParserMixin, ApidaeBaseTrekkingParser):
                 'ALLER_ITINERANCE': 'Traversée',
             }
         },
-        'accessibilities': {'create': True}
+        'accessibilities': {'create': True},
+        'geom': {'required': True},
     }
     non_fields = {
         'attachments': 'illustrations'
@@ -618,10 +621,18 @@ def filter_eid(self, src, val):
         return str(val)
 
     def filter_geom(self, src, val):
-        plan = self._find_gpx_plan_in_multimedia_items(val)
-        gpx = self._fetch_gpx_from_url(plan)
-
-        return ApidaeTrekParser._get_geom_from_gpx(gpx)
+        supported_extensions = ['gpx', 'kml', 'kmz']
+        plan = self._find_first_plan_with_supported_file_extension(val, supported_extensions)
+        geom_file = self._fetch_geometry_file(plan)
+
+        ext = plan['traductionFichiers'][0]['extension']
+        if ext == 'gpx':
+            return ApidaeTrekParser._get_geom_from_gpx(geom_file)
+        elif ext == 'kml':
+            return ApidaeTrekParser._get_geom_from_kml(geom_file)
+        elif ext == 'kmz':
+            kml_file = zipfile.ZipFile(io.BytesIO(geom_file)).read('doc.kml')
+            return ApidaeTrekParser._get_geom_from_kml(kml_file)
 
     def filter_labels(self, src, val):
         typologies, environnements = val
@@ -794,10 +805,8 @@ def _finalize_related_treks_association(self):
                 otc.save()
                 order += 1
 
-    def _fetch_gpx_from_url(self, plan):
+    def _fetch_geometry_file(self, plan):
         ref_fichier_plan = plan['traductionFichiers'][0]
-        if ref_fichier_plan['extension'] != 'gpx':
-            raise RowImportError("Le plan de l'itinéraire APIDAE n'est pas au format GPX")
         response = self.request_or_retry(url=ref_fichier_plan['url'])
         return response.content
 
@@ -865,13 +874,17 @@ def _make_marking_description(cls, itineraire):
         return marking_description
 
     @staticmethod
-    def _find_gpx_plan_in_multimedia_items(items):
-        plans = list(filter(lambda item: item['type'] == 'PLAN', items))
-        if len(plans) > 1:
-            raise RowImportError("APIDAE Trek has more than one map defined")
-        if len(plans) == 0:
-            raise RowImportError("APIDAE Trek has no map defined")
-        return plans[0]
+    def _find_first_plan_with_supported_file_extension(items, supported_extensions):
+        plans = [item for item in items if item['type'] == 'PLAN']
+        if not plans:
+            raise RowImportError('The trek from APIDAE has no attachment with the type "PLAN"')
+        supported_plans = [plan for plan in plans if plan['traductionFichiers'][0]['extension'] in supported_extensions]
+        if not supported_plans:
+            raise RowImportError(
+                "The trek from APIDAE has no attached \"PLAN\" in a supported format. "
+                f"Supported formats are : {', '.join(supported_extensions)}"
+            )
+        return supported_plans[0]
 
     @staticmethod
     def _get_geom_from_gpx(data):
@@ -889,6 +902,34 @@ def _get_geom_from_gpx(data):
             geos.transform(settings.SRID)
             return geos
 
+    @staticmethod
+    def _get_geom_from_kml(data):
+        """Given KML data as bytes it returns a geom."""
+
+        def get_geos_linestring(datasource):
+            layer = datasource[0]
+            geom = get_first_geom_with_type_in(types=['MultiLineString', 'LineString'], geoms=layer.get_geoms())
+            geom.coord_dim = 2
+            geos = geom.geos
+            if geos.geom_type == 'MultiLineString':
+                geos = geos.merged
+            return geos
+
+        def get_first_geom_with_type_in(types, geoms):
+            for g in geoms:
+                for t in types:
+                    if g.geom_type.name.startswith(t):
+                        return g
+            raise RowImportError('The attached KML geometry does not have any LineString or MultiLineString data')
+
+        with NamedTemporaryFile(mode='w+b', dir=settings.TMP_DIR) as ntf:
+            ntf.write(data)
+            ntf.flush()
+            ds = DataSource(ntf.name)
+            geos = get_geos_linestring(ds)
+            geos.transform(settings.SRID)
+            return geos
+
     @staticmethod
     def _get_layer(datasource, layer_name):
         for layer in datasource:

geotrek/trekking/tests/data/apidae_trek_parser/trace.kml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<kml xmlns="http://earth.google.com/kml/2.0">
+    <Document>
+        <name>A test trace in KML format</name>
+        <Style id="roadStyle">
+            <LineStyle>
+                <color>9F0000FF</color>
+                <width>4</width>
+            </LineStyle>
+
+        </Style>
+        <Placemark>
+            <name>A trace for my test trek</name>
+            <LookAt>
+                <longitude>6.51847</longitude>
+                <latitude>45.80873</latitude>
+            </LookAt>
+            <Point>
+                <coordinates>6.51847,45.80873,0</coordinates>
+            </Point>
+        </Placemark>
+        <Placemark>
+            <name>route</name>
+            <styleUrl>#roadStyle</styleUrl>
+            <MultiGeometry>
+                <LineString>
+                    <coordinates>6.51847,45.80873,0 6.51951,45.80845,0 6.52006,45.80845,0 6.52025,45.8081,0
+                        6.52135,45.80823,0 6.52302,45.80824,0 6.52391,45.80807,0 6.52423,45.80782,0 6.52498,45.80745,0
+                        6.52543,45.8073,0 6.52576,45.80714,0 6.52663,45.80682,0 6.527,45.8067,0 6.52755,45.80661,0
+                        6.52811,45.80627,0 6.52837,45.80598,0 6.52927,45.8056,0 6.53037,45.80508,0 6.53074,45.80463,0
+                        6.5317,45.8046,0 6.53202,45.80476,0 6.53295,45.80513,0 6.53267,45.80549,0 6.53168,45.80681,0
+                        6.53088,45.80801,0 6.53041,45.80841,0 6.53049,45.80898,0 6.5306,45.80925,0 6.53169,45.81012,0
+                        6.53268,45.81048,0 6.53422,45.81124,0 6.53541,45.81172,0 6.53551,45.81191,0 6.53585,45.81226,0
+                        6.53672,45.81271,0 6.53709,45.81267,0 6.53769,45.81285,0 6.53772,45.81319,0 6.53764,45.81339,0
+                        6.53709,45.81315,0 6.53627,45.81327,0 6.53543,45.81332,0 6.53458,45.81311,0 6.53409,45.81281,0
+                        6.53314,45.8125,0 6.53231,45.81216,0 6.53161,45.81192,0 6.53058,45.81181,0 6.52993,45.81184,0
+                        6.52865,45.81212,0 6.52885,45.81227,0 6.52889,45.81248,0 6.52823,45.81268,0 6.52652,45.81266,0
+                        6.52548,45.81239,0 6.52347,45.81166,0 6.52269,45.81117,0 6.52196,45.81094,0 6.52076,45.81031,0
+                        6.51953,45.80958,0 6.51849,45.80888,0
+                    </coordinates>
+                </LineString>
+            </MultiGeometry>
+        </Placemark>
+    </Document>
+</kml>

geotrek/trekking/tests/data/apidae_trek_parser/trace_with_no_line.kml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<kml xmlns="http://earth.google.com/kml/2.0">
+    <Document>
+        <name>A test trace in KML format</name>
+        <Style id="roadStyle">
+            <LineStyle>
+                <color>9F0000FF</color>
+                <width>4</width>
+            </LineStyle>
+
+        </Style>
+        <Placemark>
+            <name>A trace for my test trek</name>
+            <LookAt>
+                <longitude>6.51847</longitude>
+                <latitude>45.80873</latitude>
+            </LookAt>
+            <Point>
+                <coordinates>6.51847,45.80873,0</coordinates>
+            </Point>
+        </Placemark>
+    </Document>
+</kml>

geotrek/trekking/tests/data/apidae_trek_parser/trek_no_multimedia_attachments_error.json

@@ -0,0 +1,11 @@
+{
+  "numFound": 1,
+  "objetsTouristiques": [
+    {
+      "id": 123123,
+      "nom": {
+        "libelleFr": "Une belle randonnée de test mais sans fichiers multimedias liés, donc sans plan"
+      }
+    }
+  ]
+}

geotrek/trekking/tests/data/apidae_trek_parser/trek_no_plan_error.json

@@ -3,10 +3,20 @@
   "objetsTouristiques": [
     {
       "id": 123123,
-      "multimedias": [],
+      "multimedias": [
+        {
+          "type": "DOCUMENT",
+          "traductionFichiers": [
+            {
+              "url": "https://example.net/some_document.pdf",
+              "extension": "pdf"
+            }
+          ]
+        }
+      ],
       "nom": {
-        "libelleFr": "Une belle randonnée de test sans plan..."
+        "libelleFr": "Une belle randonnée de test mais sans plan"
       }
     }
   ]
-}
+}

geotrek/trekking/tests/data/apidae_trek_parser/trek_multiple_plans_error.json → geotrek/trekking/tests/data/apidae_trek_parser/trek_no_supported_plan_format_error.json

@@ -8,24 +8,24 @@
           "type": "PLAN",
           "traductionFichiers": [
             {
-              "url": "https://example.net/un_plan.gpx",
-              "extension": "gpx"
+              "url": "https://example.net/un_plan.foo",
+              "extension": "foo"
             }
           ]
         },
         {
           "type": "PLAN",
           "traductionFichiers": [
             {
-              "url": "https://example.net/et_un_autre_plan.gpx",
-              "extension": "gpx"
+              "url": "https://example.net/et_un_autre_plan.bar",
+              "extension": "bar"
             }
           ]
         }
       ],
       "nom": {
-        "libelleFr": "Une belle randonnée de test mais avec trop de plans"
+        "libelleFr": "Une belle randonnée de test mais avec des plans aux formats non supportés"
       }
     }
   ]
-}
+}

geotrek/trekking/tests/data/apidae_trek_parser/trek_with_kml_trace.json

@@ -0,0 +1,22 @@
+{
+  "numFound": 1,
+  "objetsTouristiques": [
+    {
+      "id": 123123,
+      "multimedias": [
+        {
+          "type": "PLAN",
+          "traductionFichiers": [
+            {
+              "url": "https://example.net/trace.kml",
+              "extension": "kml"
+            }
+          ]
+        }
+      ],
+      "nom": {
+        "libelleFr": "Une belle randonnée de test avec un plan au format KML"
+      }
+    }
+  ]
+}

geotrek/trekking/tests/data/apidae_trek_parser/trek_plan_no_gpx_error.json → geotrek/trekking/tests/data/apidae_trek_parser/trek_with_kmz_trace.json

@@ -8,14 +8,14 @@
           "type": "PLAN",
           "traductionFichiers": [
             {
-              "url": "https://example.net/un_plan.kmz",
+              "url": "https://example.net/trace.kmz",
               "extension": "kmz"
             }
           ]
         }
       ],
       "nom": {
-        "libelleFr": "Une belle randonnée de test avec un plan... mais pas au format gpx"
+        "libelleFr": "Une belle randonnée de test avec un plan au format KMZ (zipped KML)"
       }
     }
   ]