Merge pull request #3196 from GeotrekCE/add_aggregator_parser

Add aggregator parser

docs/changelog.rst

@@ -5,6 +5,17 @@ CHANGELOG
 2.86.0+dev (XXXX-XX-XX)
 -----------------------
 
+**New features**
+
+- Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk,
+  Path, Trail, Course, Site, SensitiveArea (#3189)
+- Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure)
+- Add aggregator parser with a conductor using json file
+
+**Bug fixes**
+
+- Fix filtering on Services List does not filter
+
 **Minor improvements**
 
 - Disable debug log in debian package post installation script.

docs/install/advanced-configuration.rst

@@ -1717,6 +1717,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "min_elevation",
         "max_elevation",
         "uuid",
+        "provider"
     ]
     COLUMNS_LISTS["workmanagementedge_export"] = [
         "eid",
@@ -1742,6 +1743,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "maintenance_difficulty",
         "published",
         "uuid",
+        "eid",
+        "provider"
     ]
     COLUMNS_LISTS["signage_view"] = [
         "code",
@@ -1834,6 +1837,9 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "reservation_id",
         "portal",
         "uuid",
+        "eid",
+        "eid2",
+        "provider"
     ]
     COLUMNS_LISTS["poi_view"] = [
         "structure",
@@ -1891,6 +1897,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "date_update",
         "date_insert",
         "uuid",
+        "eid",
+        "provider"
     ]
     COLUMNS_LISTS["touristic_event_view"] = [
         "structure",
@@ -1918,6 +1926,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "date_update",
         "date_insert",
         "uuid",
+        "eid",
+        "provider"
     ]
     COLUMNS_LISTS["feedback_view"] = [
         "email",
@@ -2163,6 +2173,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "usage_difficulty",
         "maintenance_difficulty"
         "uuid",
+        "eid",
+        "provider"
     ]
     COLUMNS_LISTS["signage_export"] = [
         "structure",
@@ -2189,6 +2201,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "min_elevation",
         "max_elevation",
         "uuid",
+        "eid",
+        "provider"
     ]
     COLUMNS_LISTS["intervention_export"] = [
         "name",
@@ -2305,6 +2319,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "max_elevation",
         "slope",
         "uuid",
+        "provider"
     ]
     COLUMNS_LISTS["poi_export"] = [
         "structure",
@@ -2392,6 +2407,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "areas",
         "approved",
         "uuid",
+        "provider"
     ]
     COLUMNS_LISTS["touristic_event_export"] = [
         "structure",
@@ -2428,6 +2444,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co
         "areas",
         "approved",
         "uuid",
+        "provider"
     ]
     COLUMNS_LISTS["feedback_export"] = [
         "email",

docs/install/import.rst

@@ -195,6 +195,45 @@ Start import from Geotrek-admin UI
 Open the top right menu and clic on ``imports``.
 
 
+Import data from a remote Geotrek instance
+==========================================
+
+Importing from a Geotrek instance works the same way as from SIT.
+A usecase for this is to aggregate data from several Geotrek-admin instance.
+
+.. danger::
+    Importing data from a remote Geotrek instance does not work with dynamic segmentation, your instance where you import data
+    must have dynamic segmentation disabled.
+
+
+For example, to import treks from another instance,
+edit ``/opt/geotrek-admin/var/conf/parsers.py`` file with the following content:
+
+::
+
+    class DemoGeotrekTrekParser(BaseGeotrekTrekParser):
+        url = "https://remote-geotrek-admin.net"  # replace url with remote instance url
+        delete = False
+        field_options = {
+            'difficulty': {'create': True, },
+            'route': {'create': True, },
+            'themes': {'create': True},
+            'practice': {'create': True},
+            'accessibilities': {'create': True},
+            'networks': {'create': True},
+            'geom': {'required': True},
+            'labels': {'create': True},
+        }
+
+Then run in command line
+
+::
+
+    sudo geotrek import DemoGeotrekTrekParser
+
+Treks are now imported into your own instance.
+
+
 Import data from a file
 =======================
 

geotrek/api/tests/test_v2.py

@@ -66,18 +66,18 @@
     'external_id', 'gpx', 'information_desks', 'kml', 'labels', 'length_2d',
     'length_3d', 'max_elevation', 'min_elevation', 'name', 'networks',
     'next', 'parents', 'parking_location', 'pdf', 'points_reference',
-    'portal', 'practice', 'previous', 'public_transport', 'published', 'ratings', 'ratings_description',
+    'portal', 'practice', 'previous', 'public_transport', 'provider', 'published', 'ratings', 'ratings_description',
     'reservation_system', 'reservation_id', 'route', 'second_external_id', 'source', 'structure',
     'themes', 'update_datetime', 'url', 'uuid', 'web_links'
 ])
 
-PATH_PROPERTIES_GEOJSON_STRUCTURE = sorted(['comments', 'length_2d', 'length_3d', 'name', 'url', 'uuid'])
+PATH_PROPERTIES_GEOJSON_STRUCTURE = sorted(['comments', 'length_2d', 'length_3d', 'name', 'provider', 'url', 'uuid'])
 
 TOUR_PROPERTIES_GEOJSON_STRUCTURE = sorted(TREK_PROPERTIES_GEOJSON_STRUCTURE + ['count_children', 'steps'])
 
 POI_PROPERTIES_GEOJSON_STRUCTURE = sorted([
     'id', 'create_datetime', 'description', 'external_id',
-    'name', 'attachments', 'published', 'type', 'type_label', 'type_pictogram',
+    'name', 'attachments', 'published', 'provider', 'type', 'type_label', 'type_pictogram',
     'update_datetime', 'url', 'uuid'
 ])
 
@@ -92,7 +92,7 @@
 TOURISTIC_CONTENT_DETAIL_JSON_STRUCTURE = sorted([
     'accessibility', 'approved', 'attachments', 'category', 'cities', 'contact', 'create_datetime', 'description',
     'description_teaser', 'departure_city', 'email', 'external_id', 'geometry', 'id', 'label_accessibility', 'name', 'pdf',
-    'portal', 'practical_info', 'published', 'reservation_id', 'reservation_system',
+    'portal', 'practical_info', 'provider', 'published', 'reservation_id', 'reservation_system',
     'source', 'structure', 'themes', 'types', 'update_datetime', 'url', 'uuid', 'website',
 ])
 
@@ -136,7 +136,7 @@
 
 INFORMATION_DESK_PROPERTIES_JSON_STRUCTURE = sorted([
     'id', 'accessibility', 'description', 'email', 'label_accessibility', 'latitude', 'longitude',
-    'municipality', 'name', 'phone', 'photo_url', 'uuid',
+    'municipality', 'name', 'phone', 'photo_url', 'provider', 'uuid',
     'postal_code', 'street', 'type', 'website'
 ])
 
@@ -147,7 +147,7 @@
 SITE_PROPERTIES_JSON_STRUCTURE = sorted([
     'accessibility', 'advice', 'ambiance', 'attachments', 'children', 'cities', 'courses', 'description', 'description_teaser', 'eid',
     'geometry', 'id', 'information_desks', 'labels', 'managers', 'name', 'orientation', 'parent', 'period', 'portal',
-    'practice', 'pdf', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', 'wind', 'web_links',
+    'practice', 'provider', 'pdf', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', 'wind', 'web_links',
 ])
 
 OUTDOORPRACTICE_PROPERTIES_JSON_STRUCTURE = sorted(['id', 'name', 'sector', 'pictogram'])
@@ -158,7 +158,7 @@
 
 SENSITIVE_AREA_PROPERTIES_JSON_STRUCTURE = sorted([
     'id', 'contact', 'create_datetime', 'description', 'elevation', 'geometry',
-    'info_url', 'kml_url', 'name', 'period', 'practices', 'published', 'species_id',
+    'info_url', 'kml_url', 'name', 'period', 'practices', 'provider', 'published', 'species_id',
     'structure', 'update_datetime', 'url'
 ])
 
@@ -172,7 +172,7 @@
 COURSE_PROPERTIES_JSON_STRUCTURE = sorted([
     'accessibility', 'advice', 'cities', 'description', 'eid', 'equipment', 'geometry', 'height', 'id',
     'length', 'name', 'ratings', 'ratings_description', 'sites', 'structure',
-    'type', 'url', 'attachments', 'max_elevation', 'min_elevation', 'parents',
+    'type', 'url', 'attachments', 'max_elevation', 'min_elevation', 'parents', 'provider',
     'pdf', 'points_reference', 'children', 'duration', 'gear', 'uuid'
 ])
 
@@ -181,7 +181,7 @@
 ORGANISM_PROPERTIES_JSON_STRUCTURE = sorted(['id', 'name'])
 
 SERVICE_DETAIL_JSON_STRUCTURE = sorted([
-    'id', 'eid', 'geometry', 'structure', 'type', 'uuid'
+    'id', 'eid', 'geometry', 'provider', 'structure', 'type', 'uuid'
 ])
 
 SERVICE_TYPE_DETAIL_JSON_STRUCTURE = sorted([
@@ -190,7 +190,7 @@
 
 INFRASTRUCTURE_DETAIL_JSON_STRUCTURE = sorted([
     'id', 'accessibility', 'attachments', 'condition', 'description', 'eid', 'geometry',
-    'implantation_year', 'maintenance_difficulty', 'name', 'structure',
+    'implantation_year', 'maintenance_difficulty', 'name', 'provider', 'structure',
     'type', 'usage_difficulty', 'uuid'
 ])
 
@@ -214,7 +214,7 @@
     'id', 'accessibility', 'approved', 'attachments', 'begin_date', 'booking', 'cities', 'contact', 'create_datetime',
     'description', 'description_teaser', 'duration', 'email', 'end_date', 'external_id', 'geometry',
     'meeting_point', 'meeting_time', 'name', 'organizer', 'participant_number', 'pdf', 'portal',
-    'practical_info', 'published', 'source', 'speaker', 'structure', 'target_audience', 'themes',
+    'practical_info', 'provider', 'published', 'source', 'speaker', 'structure', 'target_audience', 'themes',
     'type', 'update_datetime', 'url', 'uuid', 'website'
 ])
 
@@ -225,7 +225,7 @@
 SIGNAGE_DETAIL_JSON_STRUCTURE = sorted([
     'id', 'attachments', 'blades', 'code', 'condition', 'description', 'eid',
     'geometry', 'implantation_year', 'name', 'printed_elevation', 'sealing',
-    'structure', 'type', 'uuid'
+    'provider', 'structure', 'type', 'uuid'
 ])
 
 SIGNAGE_TYPE_DETAIL_JSON_STRUCTURE = sorted([

geotrek/api/v2/serializers.py

@@ -181,7 +181,7 @@ class ServiceSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
 
         class Meta:
             model = trekking_models.Service
-            fields = ('id', 'eid', 'geometry', 'structure', 'type', 'uuid')
+            fields = ('id', 'eid', 'geometry', 'provider', 'structure', 'type', 'uuid')
 
 
 class ReservationSystemSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
@@ -424,7 +424,7 @@ class Meta:
                 'id', 'accessibility', 'attachments', 'approved', 'category', 'description',
                 'description_teaser', 'departure_city', 'geometry', 'label_accessibility',
                 'practical_info', 'url', 'cities', 'create_datetime',
-                'external_id', 'name', 'pdf', 'portal', 'published',
+                'external_id', 'name', 'pdf', 'portal', 'provider', 'published',
                 'source', 'structure', 'themes',
                 'update_datetime', 'types', 'contact', 'email',
                 'website', 'reservation_system', 'reservation_id', 'uuid'
@@ -461,7 +461,7 @@ class Meta:
                 'cities', 'contact', 'create_datetime', 'description', 'description_teaser',
                 'duration', 'email', 'end_date', 'external_id', 'geometry', 'meeting_point',
                 'meeting_time', 'name', 'organizer', 'participant_number', 'pdf', 'portal',
-                'practical_info', 'published', 'source', 'speaker', 'structure',
+                'practical_info', 'published', 'provider', 'source', 'speaker', 'structure',
                 'target_audience', 'themes', 'type', 'update_datetime', 'url', 'uuid', 'website'
             )
 
@@ -500,7 +500,7 @@ class Meta:
             fields = (
                 'id', 'accessibility', 'description', 'email', 'label_accessibility', 'latitude', 'longitude',
                 'municipality', 'name', 'phone', 'photo_url', 'uuid',
-                'postal_code', 'street', 'type', 'website'
+                'postal_code', 'provider', 'street', 'type', 'website'
             )
 
 
@@ -518,7 +518,7 @@ class Meta:
             model = core_models.Path
             fields = (
                 'id', 'comments', 'geometry', 'length_2d', 'length_3d',
-                'name', 'url', 'uuid'
+                'name', 'provider', 'url', 'uuid'
             )
 
 
@@ -699,7 +699,7 @@ class Meta:
                 'information_desks', 'kml', 'labels', 'length_2d', 'length_3d',
                 'max_elevation', 'min_elevation', 'name', 'networks', 'next',
                 'parents', 'parking_location', 'pdf', 'points_reference',
-                'portal', 'practice', 'ratings', 'ratings_description', 'previous', 'public_transport',
+                'portal', 'practice', 'provider', 'ratings', 'ratings_description', 'previous', 'public_transport',
                 'published', 'reservation_system', 'reservation_id', 'route', 'second_external_id',
                 'source', 'structure', 'themes', 'update_datetime', 'url', 'uuid', 'web_links'
             )
@@ -764,9 +764,9 @@ class Meta:
             model = trekking_models.POI
             fields = (
                 'id', 'description', 'external_id',
-                'geometry', 'name', 'attachments', 'published', 'type',
+                'geometry', 'name', 'attachments', 'provider', 'published', 'type',
                 'type_label', 'type_pictogram', 'url', 'uuid',
-                'create_datetime', 'update_datetime',
+                'create_datetime', 'update_datetime'
             )
 
     class ThemeSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
@@ -845,7 +845,7 @@ class Meta:
             fields = (
                 'id', 'contact', 'create_datetime', 'description', 'elevation',
                 'geometry', 'info_url', 'kml_url', 'name', 'period',
-                'practices', 'published', 'species_id', 'structure',
+                'practices', 'published', 'species_id', 'provider', 'structure',
                 'update_datetime', 'url'
             )
 
@@ -1055,7 +1055,7 @@ class Meta:
             fields = (
                 'id', 'accessibility', 'advice', 'ambiance', 'attachments', 'cities', 'children', 'description',
                 'description_teaser', 'eid', 'geometry', 'information_desks', 'labels', 'managers',
-                'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice',
+                'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice', 'provider',
                 'ratings', 'sector', 'source', 'structure', 'themes',
                 'type', 'url', 'uuid', 'courses', 'web_links', 'wind',
             )
@@ -1115,7 +1115,7 @@ class Meta:
             fields = (
                 'id', 'accessibility', 'advice', 'attachments', 'children', 'cities', 'description', 'duration', 'eid',
                 'equipment', 'gear', 'geometry', 'height', 'length', 'max_elevation',
-                'min_elevation', 'name', 'parents', 'pdf', 'points_reference', 'ratings', 'ratings_description',
+                'min_elevation', 'name', 'parents', 'pdf', 'points_reference', 'provider', 'ratings', 'ratings_description',
                 'sites', 'structure', 'type', 'url', 'uuid'
             )
 
@@ -1209,7 +1209,7 @@ def get_accessibility(self, obj):
         class Meta:
             model = infrastructure_models.Infrastructure
             fields = ('id', 'accessibility', 'attachments', 'condition', 'description', 'eid', 'geometry', 'name',
-                      'implantation_year', 'maintenance_difficulty', 'structure', 'type', 'usage_difficulty', 'uuid')
+                      'implantation_year', 'maintenance_difficulty', 'provider', 'structure', 'type', 'usage_difficulty', 'uuid')
 
     class InfrastructureConditionSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
 
@@ -1254,7 +1254,7 @@ class SignageSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
         class Meta:
             model = signage_models.Signage
             fields = ('id', 'attachments', 'blades', 'code', 'condition', 'description', 'eid',
-                      'geometry', 'implantation_year', 'name', 'printed_elevation', 'sealing',
+                      'geometry', 'implantation_year', 'name', 'printed_elevation', 'provider', 'sealing',
                       'structure', 'type', 'uuid')
 
     class SignageTypeSerializer(DynamicFieldsMixin, serializers.ModelSerializer):

geotrek/common/management/commands/import.py

@@ -12,7 +12,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('parser', help='Parser class name in var/conf/parsers.py (or dotted syntax in python path)')
-        parser.add_argument('shapefile', nargs="?")
+        parser.add_argument('filename', nargs="?", help='Optional file used to feed database')
         parser.add_argument('-l', dest='limit', type=int, help='Limit number of lines to import')
         parser.add_argument('--encoding', '-e', default='utf8')
 
@@ -41,7 +41,7 @@ def handle(self, *args, **options):
             Parser = getattr(module, class_name)
         except AttributeError:
             raise CommandError("Failed to import parser class '{0}'".format(class_name))
-        if not Parser.filename and not Parser.url and not options['shapefile']:
+        if not Parser.filename and not Parser.url and not options['filename']:
             raise CommandError("File path missing")
 
         def progress_cb(progress, line, eid):
@@ -52,7 +52,7 @@ def progress_cb(progress, line, eid):
         parser = Parser(progress_cb=progress_cb, encoding=encoding)
 
         try:
-            parser.parse(options['shapefile'], limit=limit)
+            parser.parse(options['filename'], limit=limit)
         except ImportError as e:
             raise CommandError(e)