From 8c33660772a2cb60ecbbe98967ecfcc08f9feca8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 17:03:25 +0200 Subject: [PATCH 001/116] Change name option filename --- geotrek/common/management/commands/import.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geotrek/common/management/commands/import.py b/geotrek/common/management/commands/import.py index 30d3581c14..c58419b75f 100644 --- a/geotrek/common/management/commands/import.py +++ b/geotrek/common/management/commands/import.py @@ -12,7 +12,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('parser', help='Parser class name in var/conf/parsers.py (or dotted syntax in python path)') - parser.add_argument('shapefile', nargs="?") + parser.add_argument('filename', nargs="?") parser.add_argument('-l', dest='limit', type=int, help='Limit number of lines to import') parser.add_argument('--encoding', '-e', default='utf8') @@ -41,7 +41,7 @@ def handle(self, *args, **options): Parser = getattr(module, class_name) except AttributeError: raise CommandError("Failed to import parser class '{0}'".format(class_name)) - if not Parser.filename and not Parser.url and not options['shapefile']: + if not Parser.filename and not Parser.url and not options['filename']: raise CommandError("File path missing") def progress_cb(progress, line, eid): @@ -52,7 +52,7 @@ def progress_cb(progress, line, eid): parser = Parser(progress_cb=progress_cb, encoding=encoding) try: - parser.parse(options['shapefile'], limit=limit) + parser.parse(options['filename'], limit=limit) except ImportError as e: raise CommandError(e) From 59f91c5b2057c93e3196a01dd877bdd92ee7d191 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 17:05:04 +0200 Subject: [PATCH 002/116] Improve actual parsers with mapping --- geotrek/common/parsers.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 8465580716..fc4929cccf 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -353,12 +353,16 @@ def get_mapping(self, src, val, mapping, partial): found = True break if not found: - self.add_warning(_("Bad value '{val}' for field {src}. Should contain {values}").format(val=val, src=src, separator=self.separator, values=', '.join(mapping.keys()))) + values = [str(key) for key in mapping.keys()] + self.add_warning(_("Bad value '{val}' for field {src}. Should contain {values}").format(val=str(val), src=src, separator=self.separator, values=values)) return None else: if mapping is not None: - if val not in mapping.keys(): - self.add_warning(_("Bad value '{val}' for field {src}. Should be {values}").format(val=val, src=src, separator=self.separator, values=', '.join(mapping.keys()))) + if val and val not in mapping.keys(): + values = [str(key) for key in mapping.keys()] + self.add_warning(_("Bad value '{val}' for field {src}. Should be {values}").format(val=str(val), src=src, separator=self.separator, values=values)) + return None + if not val: return None val = mapping[val] return val @@ -388,7 +392,8 @@ def filter_m2m(self, src, val, model, field, mapping=None, partial=False, create val = val.split(self.separator) dst = [] for subval in val: - subval = subval.strip() + if isinstance(subval, str): + subval = subval.strip() subval = self.get_mapping(src, subval, mapping, partial) if subval is None: continue @@ -449,7 +454,7 @@ def parse(self, filename=None, limit=None): if filename: self.filename = filename if not self.url and not self.filename: - raise GlobalImportError(_("Filename is required")) + raise GlobalImportError(_("Filename or url is required")) if self.filename and not os.path.exists(self.filename): raise GlobalImportError(_("File does not exists at: {filename}").format(filename=self.filename)) self.start() From 2b5c25706ca7c6fed4b91cf0483971751a0136b4 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 17:05:44 +0200 Subject: [PATCH 003/116] Add GeotrekParser --- geotrek/common/parsers.py | 100 +++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index fc4929cccf..4d11e0932f 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1,4 +1,5 @@ from io import BytesIO +import json import os import re import requests @@ -16,11 +17,14 @@ from os.path import dirname from urllib.parse import urlparse +from django.contrib.gis.geos import GEOSGeometry, WKBWriter +from django.core.management import call_command from django.db import models, connection from django.db.utils import DatabaseError from django.contrib.auth import get_user_model from django.contrib.gis.gdal import DataSource, GDALException, CoordTransform -from django.contrib.gis.geos import Point +from django.contrib.gis.geos import Point, Polygon +from django.core.exceptions import ImproperlyConfigured from django.core.files.base import ContentFile from django.template.loader import render_to_string from django.utils import translation @@ -865,3 +869,97 @@ def next_row(self): def normalize_field_name(self, name): return name + + +class GeotrekParser(AttachmentParserMixin, Parser): + model = None + next_url = '' + url = None + separator = None + delete = True + eid = 'eid' + constant_fields = {} + url_categories = {} + replace_fields = {} + m2m_replace_fields = {} + categories_keys_api_v2 = {} + non_fields = { + 'attachments': "attachments", + } + field_options = { + 'geom': {'required': True}, + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and + not f.name == 'id') + self.m2m_fields = { + f.name: f.name + for f in self.model._meta.many_to_many + } + for key, value in self.replace_fields.items(): + self.fields[key] = value + + for key, value in self.m2m_replace_fields.items(): + self.m2m_fields[key] = value + self.translated_fields = [field for field in get_translated_fields(self.model)] + for category in self.url_categories.keys(): + route = self.url_categories[category] + response = self.request_or_retry(f"{self.url}{route}", ) # params=params) + self.field_options.setdefault(category, {}) + if self.categories_keys_api_v2.get(category): + if category in self.translated_fields: + + self.field_options[category]["mapping"] = { + r["id"]: r[self.categories_keys_api_v2[category]][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] for r in response.json()['results'] + } + else: + self.field_options[category]["mapping"] = { + r["id"]: r[self.categories_keys_api_v2[category]] + for r in response.json()['results'] + } + else: + raise ImproperlyConfigured(f"{category} is not configured in categories_keys_api_v2") + self.creator, created = get_user_model().objects.get_or_create(username='import', defaults={'is_active': False}) + print(self.field_options) + + def filter_attachments(self, src, val): + return [(subval.get('url'), subval.get('legend'), subval.get('author')) for subval in val] + + def apply_filter(self, dst, src, val): + val = super().apply_filter(dst, src, val) + if dst in self.translated_fields: + if isinstance(val, dict): + val = val.get(settings.MODELTRANSLATION_DEFAULT_LANGUAGE) + return val + + def normalize_field_name(self, name): + return name + + @property + def items(self): + return self.root['results'] + + def filter_geom(self, src, val): + geom = GEOSGeometry(json.dumps(val)) + geom.transform(settings.SRID) + geom = WKBWriter().write(geom) + geom = GEOSGeometry(geom) + return geom + + def next_row(self): + bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) + bbox.srid = settings.SRID + bbox.transform(4326) # WGS84 + while self.next_url: + params = { + 'in_bbox': ','.join([str(coord) for coord in bbox.extent]), + } + response = self.request_or_retry(self.next_url, params=params) + self.root = response.json() + self.nb = int(self.root['count']) + + for row in self.items: + yield row + self.next_url = self.root['next'] From e72217015d46bee52bbfdf7fa4bc353933adefaf Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 17:07:31 +0200 Subject: [PATCH 004/116] Add parser tourism from api v2 --- geotrek/tourism/parsers.py | 153 ++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index c390c7ba8b..784d447e44 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -14,7 +14,7 @@ from django.core.files.uploadedfile import UploadedFile from geotrek.common.parsers import (AttachmentParserMixin, Parser, - TourInSoftParser) + TourInSoftParser, GeotrekParser) from geotrek.tourism.models import (InformationDesk, TouristicContent, TouristicEvent, TouristicContentType1, TouristicContentType2) @@ -888,3 +888,154 @@ def filter_end_date(self, src, val): class TouristicEventTourInSoftParserV3(TouristicEventTourInSoftParser): version_tourinsoft = 3 + + +class GeotrekTouristicContentParser(GeotrekParser): + url = None + model = TouristicContent + constant_fields = { + 'published': True, + 'deleted': False, + } + + replace_fields = { + "eid": "uuid", + "geom": "geometry", + } + + m2m_replace_fields = { + "type1": "types", + "type2": "types" + } + + url_categories = { + "category": "/api/v2/touristiccontent_category/", + "themes": "/api/v2/theme/", + } + + categories_keys_api_v2 = { + 'category': 'label', + 'themes': 'label', + } + + natural_keys = { + 'category': 'label', + 'themes': 'label', + 'type1': 'label', + 'type2': 'label' + } + + field_options = { + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + response = self.request_or_retry(f"{self.url}/api/v2/touristiccontent_category/", ) + self.field_options.setdefault("type1", {}) + self.field_options.setdefault("type2", {}) + self.field_options["type1"]["mapping"] = {} + self.field_options["type2"]["mapping"] = {} + for r in response.json()['results']: + for type_category in r['types']: + if type_category["id"] % 10 == 1: + self.field_options['type1']["mapping"][type_category["id"]] = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + if type_category["id"] % 10 == 2: + self.field_options['type2']["mapping"][type_category["id"]] = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + + def next_row(self): + self.next_url = f"{self.url}/api/v2/touristiccontent" + return super().next_row() + + def filter_type1(self, src, val): + type1_result = [] + for key in val.keys(): + if int(key) % 10 == 1: + type1_result.append(int(key)) + return self.apply_filter('type1', src, type1_result) + + def filter_type2(self, src, val): + type2_result = [] + for key in val.keys(): + if int(key) % 10 == 2: + type2_result.append(int(key)) + return self.apply_filter('type2', src, type2_result) + + +class GeotrekTouristicEventParser(GeotrekParser): + url = None + model = TouristicEvent + constant_fields = { + 'published': True, + 'deleted': False, + } + replace_fields = { + "eid": "uuid", + "geom": "geometry" + } + url_categories = { + "type": "/api/v2/touristicevent_type/", + } + categories_keys_api_v2 = { + 'type': 'type', + } + natural_keys = { + 'type': 'type', + } + + field_options = { + "type": {"create": True} + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/touristicevent" + return super().next_row() + + +class GeotrekInformationDeskParser(GeotrekParser): + url = None + model = InformationDesk + constant_fields = {} + replace_fields = { + "eid": ["uuid", "id"], + "geom": ["latitude", "longitude"], + "photo": "photo_url" + } + url_categories = {} + categories_keys_api_v2 = {} + natural_keys = { + 'type': 'label', + } + + field_options = { + "type": {"create": True} + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/informationdesk" + return super().next_row() + + def filter_eid(self, src, val): + uuid, id_iddesk = val + final_value = uuid + if not uuid: + final_value = id_iddesk + return final_value + + def filter_geom(self, src, val): + lat, lng = val + return Point(lat, lng, srid=settings.API_SRID).transform(settings.SRID, clone=True) + + def filter_type(self, src, val): + return self.apply_filter('type', src, val["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE]) + + def filter_photo(self, src, val): + if not val: + return None + content = self.download_attachment(val) + if content is None: + return None + f = ContentFile(content) + basename, ext = os.path.splitext(os.path.basename(val)) + name = '%s%s' % (basename[:128], ext) + file = UploadedFile(f, name=name) + return file From c22e556062712f7fb3057b733f0cb541126ba3a8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 17:08:29 +0200 Subject: [PATCH 005/116] Add parser api v2 Geotrek Trekking --- geotrek/trekking/parsers.py | 110 +++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 3 deletions(-) diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index f466fcc345..528ac925eb 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -1,8 +1,10 @@ -from django.contrib.gis.geos import Point +import json +from django.conf import settings +from django.contrib.gis.geos import Point, GEOSGeometry from django.utils.translation import gettext as _ -from geotrek.common.parsers import ShapeParser, AttachmentParserMixin -from geotrek.trekking.models import Trek +from geotrek.common.parsers import ShapeParser, AttachmentParserMixin, GeotrekParser +from geotrek.trekking.models import POI, Service, Trek class DurationParserMixin: @@ -60,3 +62,105 @@ def filter_geom(self, src, val): self.add_warning(_("Invalid geometry type for field '{src}'. Should be LineString, not {geom_type}").format(src=src, geom_type=val.geom_type)) return None return val + + +class GeotrekTrekParser(GeotrekParser): + url = None + model = Trek + constant_fields = { + 'published': True, + 'deleted': False, + } + replace_fields = { + "eid": "uuid", + "eid2": "second_external_id", + "geom": "geometry" + } + url_categories = { + "difficulty": "/api/v2/trek_difficulty/", + "practice": "/api/v2/trek_practice/", + "route": "/api/v2/trek_route/", + "themes": "/api/v2/theme/", + "accessibilities": "/api/v2/trek_accessibility/", + "networks": "/api/v2/trek_network/" + } + categories_keys_api_v2 = { + 'difficulty': 'label', + 'route': 'route', + 'themes': 'label', + 'practice': 'name', + 'accessibilities': 'name', + 'networks': 'label' + } + natural_keys = { + 'difficulty': 'difficulty', + 'route': 'route', + 'themes': 'label', + 'practice': 'name', + 'accessibilities': 'name', + 'networks': 'network', + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/trek" + return super().next_row() + + def filter_parking_location(self, src, val): + if val: + return Point(val[0], val[1], srid=settings.API_SRID) + + def filter_points_reference(self, src, val): + if val: + geom = GEOSGeometry(json.dumps(val)) + return geom.transform(settings.SRID, clone=True) + + +class GeotrekServiceParser(GeotrekParser): + url = None + model = Service + constant_fields = { + 'deleted': False, + } + replace_fields = { + "eid": "uuid", + "geom": "geometry" + } + url_categories = { + "type": "/api/v2/service_type/", + } + categories_keys_api_v2 = { + 'type': 'name', + } + natural_keys = { + 'type': 'name' + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/service" + return super().next_row() + + +class GeotrekPOIParser(GeotrekParser): + url = None + model = POI + constant_fields = { + 'published': True, + 'deleted': False, + } + replace_fields = { + "eid": "uuid", + "geom": "geometry" + } + url_categories = { + "type": "/api/v2/poi_type/", + } + categories_keys_api_v2 = { + 'type': 'label', + } + natural_keys = { + 'type': 'label', + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/poi" + return super().next_row() From f25346c8daa6aa20910e005f2b20bfe253b87f63 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 17:09:08 +0200 Subject: [PATCH 006/116] Add parsers infrastructure signage Api v2 --- geotrek/infrastructure/parsers.py | 34 ++++++++++++++++++++++++++++ geotrek/signage/parsers.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 geotrek/infrastructure/parsers.py create mode 100644 geotrek/signage/parsers.py diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py new file mode 100644 index 0000000000..9fcc5b3316 --- /dev/null +++ b/geotrek/infrastructure/parsers.py @@ -0,0 +1,34 @@ +from geotrek.common.parsers import GeotrekParser +from geotrek.infrastructure.models import Infrastructure + + +class GeotrekInfrastructureParser(GeotrekParser): + url = None + model = Infrastructure + constant_fields = { + "published": True, + "deleted": False + } + replace_fields = { + "eid": "uuid" + } + url_categories = { + 'condition': '/api/v2/infrastructure_condition/', + 'type': '/api/v2/infrastructure_type/', + } + categories_keys_api_v2 = { + 'condition': 'label', + 'type': 'label' + } + natural_keys = { + 'condition': 'label', + 'type': 'label' + } + + field_options = { + "type": {"create": True} + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/infrastructure" + return super().next_row() diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py new file mode 100644 index 0000000000..f7befc4e71 --- /dev/null +++ b/geotrek/signage/parsers.py @@ -0,0 +1,37 @@ +from geotrek.common.parsers import GeotrekParser +from geotrek.signage.models import Signage + + +class GeotrekSignageParser(GeotrekParser): + url = None + model = Signage + constant_fields = { + "published": True, + "deleted": False + } + replace_fields = { + "eid": "uuid" + } + url_categories = { + 'sealing': '/api/v2/signage_sealing/', + 'condition': '/api/v2/infrastructure_condition/', + 'type': '/api/v2/signage_type/', + } + categories_keys_api_v2 = { + 'condition': 'label', + 'sealing': 'label', + 'type': 'label' + } + natural_keys = { + 'condition': 'label', + 'sealing': 'label', + 'type': 'label' + } + + field_options = { + "type": {"create": True} + } + + def next_row(self): + self.next_url = f"{self.url}/api/v2/signage" + return super().next_row() From 303d6ae36f82ae205b9b0c31fa93f12656b4366b Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 19:12:41 +0200 Subject: [PATCH 007/116] Fix test task raise message --- geotrek/common/tests/test_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/tests/test_tasks.py b/geotrek/common/tests/test_tasks.py index 60a6c73a4a..b9a0916645 100644 --- a/geotrek/common/tests/test_tasks.py +++ b/geotrek/common/tests/test_tasks.py @@ -39,7 +39,7 @@ def test_import_datas_from_web_message_exception(self): def test_import_datas_from_web_other_exception(self): self.assertRaisesMessage( GlobalImportError, - 'Filename is required', + 'Filename or url is required', import_datas_from_web, name='OrganismParser', module='geotrek.common.tests.test_tasks' From e933726d21ad0fb554c5c1c33575e7e8a727117d Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 28 Jul 2022 19:13:28 +0200 Subject: [PATCH 008/116] Improve attachment parser, separate save_attachments --- geotrek/common/parsers.py | 127 +++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 51 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 4d11e0932f..3b5e0ad566 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -630,69 +630,94 @@ def download_attachment(self, url): return response.content return None - def save_attachments(self, src, val): - updated = False - attachments_to_delete = list(Attachment.objects.attachments_for_object(self.obj)) + def check_attachment_updated(self, attachments_to_delete, updated, **kwargs): + found = False + for attachment in attachments_to_delete: + upload_name, ext = os.path.splitext(attachment_upload(attachment, kwargs.get('name'))) + existing_name = attachment.attachment_file.name + if re.search(r"^{name}(_[a-zA-Z0-9]{{7}})?{ext}$".format( + name=upload_name, ext=ext), existing_name + ) and not self.has_size_changed(kwargs.get('url'), attachment): + found = True + attachments_to_delete.remove(attachment) + if kwargs.get('author') != attachment.author or kwargs.get('legend') != attachment.legend: + attachment.author = kwargs.get('author') + attachment.legend = textwrap.shorten(kwargs.get('legend'), width=127) + attachment.save() + updated = True + break + return found, updated + + def generate_content_attachment(self, attachment, parsed_url, url, updated, name): + if (parsed_url.scheme in ('http', 'https') and self.download_attachments) or parsed_url.scheme == 'ftp': + content = self.download_attachment(url) + if content is None: + return False, updated + f = ContentFile(content) + if settings.PAPERCLIP_MAX_BYTES_SIZE_IMAGE and settings.PAPERCLIP_MAX_BYTES_SIZE_IMAGE < f.size: + logger.warning( + _(f'{self.obj.__class__.__name__} #{self.obj.pk} - {url} : downloaded file is too large')) + return False, updated + try: + image = Image.open(BytesIO(content)) + if settings.PAPERCLIP_MIN_IMAGE_UPLOAD_WIDTH and settings.PAPERCLIP_MIN_IMAGE_UPLOAD_WIDTH > image.width: + logger.warning( + _(f"{self.obj.__class__.__name__} #{self.obj.pk} - {url} : downloaded file is not wide enough")) + return False, updated + if settings.PAPERCLIP_MIN_IMAGE_UPLOAD_HEIGHT and settings.PAPERCLIP_MIN_IMAGE_UPLOAD_HEIGHT > image.height: + logger.warning( + _(f"{self.obj.__class__.__name__} #{self.obj.pk} - {url} : downloaded file is not tall enough")) + return False, updated + except UnidentifiedImageError: + pass + attachment.attachment_file.save(name, f, save=False) + else: + attachment.attachment_link = url + return True, updated + + def remove_attachments(self, attachments_to_delete): + if self.delete_attachments: + for att in attachments_to_delete: + att.delete() + + def generate_attachment(self, **kwargs): + attachment = Attachment() + attachment.content_object = self.obj + attachment.filetype = self.filetype + attachment.creator = self.creator + attachment.author = kwargs.get('author') + attachment.legend = textwrap.shorten(kwargs.get('legend'), width=127) + return attachment + + def generate_attachments(self, src, val, attachments_to_delete, updated): + attachments = [] for url, legend, author in self.filter_attachments(src, val): url = self.base_url + url legend = legend or "" author = author or "" basename, ext = os.path.splitext(os.path.basename(url)) name = '%s%s' % (basename[:128], ext) - found = False - for attachment in attachments_to_delete: - upload_name, ext = os.path.splitext(attachment_upload(attachment, name)) - existing_name = attachment.attachment_file.name - if re.search(r"^{name}(_[a-zA-Z0-9]{{7}})?{ext}$".format( - name=upload_name, ext=ext), existing_name - ) and not self.has_size_changed(url, attachment): - found = True - attachments_to_delete.remove(attachment) - if author != attachment.author or legend != attachment.legend: - attachment.author = author - attachment.legend = textwrap.shorten(legend, width=127) - attachment.save() - updated = True - break + found, updated = self.check_attachment_updated(attachments_to_delete, updated, name=name, url=url, + legend=legend, author=author) if found: continue parsed_url = urlparse(url) - - attachment = Attachment() - attachment.content_object = self.obj - attachment.filetype = self.filetype - attachment.creator = self.creator - attachment.author = author - attachment.legend = textwrap.shorten(legend, width=127) - - if (parsed_url.scheme in ('http', 'https') and self.download_attachments) or parsed_url.scheme == 'ftp': - content = self.download_attachment(url) - if content is None: - continue - f = ContentFile(content) - if settings.PAPERCLIP_MAX_BYTES_SIZE_IMAGE and settings.PAPERCLIP_MAX_BYTES_SIZE_IMAGE < f.size: - logger.warning(_(f'{self.obj.__class__.__name__} #{self.obj.pk} - {url} : downloaded file is too large')) - return updated - try: - image = Image.open(BytesIO(content)) - if settings.PAPERCLIP_MIN_IMAGE_UPLOAD_WIDTH and settings.PAPERCLIP_MIN_IMAGE_UPLOAD_WIDTH > image.width: - logger.warning(_(f"{self.obj.__class__.__name__} #{self.obj.pk} - {url} : downloaded file is not wide enough")) - return updated - if settings.PAPERCLIP_MIN_IMAGE_UPLOAD_HEIGHT and settings.PAPERCLIP_MIN_IMAGE_UPLOAD_HEIGHT > image.height: - logger.warning(_(f"{self.obj.__class__.__name__} #{self.obj.pk} - {url} : downloaded file is not tall enough")) - return updated - except UnidentifiedImageError: - pass - attachment.attachment_file.save(name, f, save=False) - else: - attachment.attachment_link = url - attachment.save() + attachment = self.generate_attachment(author=author, legend=legend) + save, updated = self.generate_content_attachment(attachment, parsed_url, url, updated, name) + if not save: + continue + attachments.append(attachment) updated = True + return updated, attachments - if self.delete_attachments: - for att in attachments_to_delete: - att.delete() + def save_attachments(self, src, val): + updated = False + attachments_to_delete = list(Attachment.objects.attachments_for_object(self.obj)) + updated, attachments = self.generate_attachments(src, val, attachments_to_delete, updated) + Attachment.objects.bulk_create(attachments) + + self.remove_attachments(attachments_to_delete) return updated From 3a7299c9db663a0fc2d7059c06cfb381cee12f12 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 14:28:28 +0200 Subject: [PATCH 009/116] Change categories integration datas --- geotrek/common/parsers.py | 21 +++++++++++---------- geotrek/tourism/parsers.py | 10 ++++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 3b5e0ad566..81055723bb 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -934,16 +934,17 @@ def __init__(self, *args, **kwargs): response = self.request_or_retry(f"{self.url}{route}", ) # params=params) self.field_options.setdefault(category, {}) if self.categories_keys_api_v2.get(category): - if category in self.translated_fields: - - self.field_options[category]["mapping"] = { - r["id"]: r[self.categories_keys_api_v2[category]][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] for r in response.json()['results'] - } - else: - self.field_options[category]["mapping"] = { - r["id"]: r[self.categories_keys_api_v2[category]] - for r in response.json()['results'] - } + self.field_options[category]["mapping"] = {} + results = response.json()['results'] + for result in results: + id_result = result['id'] + label = result[self.categories_keys_api_v2[category]] + if isinstance(result[self.categories_keys_api_v2[category]], dict): + if label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE]: + self.field_options[category]["mapping"][id_result] = label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + else: + if label: + self.field_options[category]["mapping"][id_result] = label else: raise ImproperlyConfigured(f"{category} is not configured in categories_keys_api_v2") self.creator, created = get_user_model().objects.get_or_create(username='import', defaults={'is_active': False}) diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 784d447e44..4562f9cb70 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -937,10 +937,12 @@ def __init__(self, *args, **kwargs): self.field_options["type2"]["mapping"] = {} for r in response.json()['results']: for type_category in r['types']: - if type_category["id"] % 10 == 1: - self.field_options['type1']["mapping"][type_category["id"]] = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] - if type_category["id"] % 10 == 2: - self.field_options['type2']["mapping"][type_category["id"]] = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + label_lang = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + id_category = type_category["id"] + if id_category % 10 == 1: + self.field_options['type1']["mapping"][id_category] = label_lang if label_lang else None + if id_category % 10 == 2: + self.field_options['type2']["mapping"][id_category] = label_lang if label_lang else None def next_row(self): self.next_url = f"{self.url}/api/v2/touristiccontent" From 4f877b94e23f39cf17f9431e7e52189e7d133827 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 15:52:53 +0200 Subject: [PATCH 010/116] Add test GeotrekTrekParser --- geotrek/common/parsers.py | 1 - geotrek/trekking/parsers.py | 2 +- geotrek/trekking/tests/test_parsers.py | 55 +++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 81055723bb..5a9683fa12 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -948,7 +948,6 @@ def __init__(self, *args, **kwargs): else: raise ImproperlyConfigured(f"{category} is not configured in categories_keys_api_v2") self.creator, created = get_user_model().objects.get_or_create(username='import', defaults={'is_active': False}) - print(self.field_options) def filter_attachments(self, src, val): return [(subval.get('url'), subval.get('legend'), subval.get('author')) for subval in val] diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 528ac925eb..7b8a9b2bcf 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -78,9 +78,9 @@ class GeotrekTrekParser(GeotrekParser): } url_categories = { "difficulty": "/api/v2/trek_difficulty/", - "practice": "/api/v2/trek_practice/", "route": "/api/v2/trek_route/", "themes": "/api/v2/theme/", + "practice": "/api/v2/trek_practice/", "accessibilities": "/api/v2/trek_accessibility/", "networks": "/api/v2/trek_network/" } diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 4865227649..2938cb819d 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -1,12 +1,16 @@ +from unittest import mock +import json import os +from unittest import skipIf +from django.conf import settings from django.contrib.gis.geos import Point, LineString, MultiLineString, WKTWriter from django.core.management import call_command from django.test import TestCase from geotrek.common.models import Theme, FileType from geotrek.trekking.models import Trek, DifficultyLevel, Route -from geotrek.trekking.parsers import TrekParser +from geotrek.trekking.parsers import TrekParser, GeotrekTrekParser class TrekParserFilterDurationTests(TestCase): @@ -124,3 +128,52 @@ def test_create(self): self.assertEqual(trek.route, self.route) self.assertQuerysetEqual(trek.themes.all(), [repr(t) for t in self.themes], ordered=False) self.assertEqual(WKTWriter(precision=4).write(trek.geom), WKT) + + +class TestGeotrekTrekParser(GeotrekTrekParser): + url = "https://test.fr" + + field_options = { + 'difficulty': {'create': True, }, + 'route': {'create': True, }, + 'themes': {'create': True}, + 'practice': {'create': True}, + 'accessibilities': {'create': True}, + 'networks': {'create': True}, + } + + +@skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') +class TrekGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.difficulty = DifficultyLevel.objects.create(difficulty="Facile") + cls.route = Route.objects.create(route="Boucle") + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek.json'] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 2) + trek = Trek.objects.all().first() + self.assertEqual(trek.name, "Loop of the pic of 3 lords") + self.assertEqual(str(trek.difficulty), 'Very easy') + self.assertEqual(str(trek.practice), 'Horse') From 382acb958e3ae0a5a0a14288857387ff95b75dde Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 16:30:11 +0200 Subject: [PATCH 011/116] Add POI test parser --- geotrek/trekking/tests/test_parsers.py | 48 +++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 2938cb819d..a21343cb31 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -7,10 +7,11 @@ from django.contrib.gis.geos import Point, LineString, MultiLineString, WKTWriter from django.core.management import call_command from django.test import TestCase +from django.test.utils import override_settings from geotrek.common.models import Theme, FileType -from geotrek.trekking.models import Trek, DifficultyLevel, Route -from geotrek.trekking.parsers import TrekParser, GeotrekTrekParser +from geotrek.trekking.models import POI, Trek, DifficultyLevel, Route +from geotrek.trekking.parsers import TrekParser, GeotrekPOIParser, GeotrekTrekParser class TrekParserFilterDurationTests(TestCase): @@ -143,12 +144,18 @@ class TestGeotrekTrekParser(GeotrekTrekParser): } +class TestGeotrekPOIParser(GeotrekPOIParser): + url = "https://test.fr" + + field_options = { + 'type': {'create': True, }, + } + + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class TrekGeotrekParserTests(TestCase): @classmethod def setUpTestData(cls): - cls.difficulty = DifficultyLevel.objects.create(difficulty="Facile") - cls.route = Route.objects.create(route="Boucle") cls.filetype = FileType.objects.create(type="Photographie") @mock.patch('requests.get') @@ -177,3 +184,36 @@ def mocked_json(): self.assertEqual(trek.name, "Loop of the pic of 3 lords") self.assertEqual(str(trek.difficulty), 'Very easy') self.assertEqual(str(trek.practice), 'Horse') + + +@skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') +class POIGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['poi_type.json', 'poi.json'] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekPOIParser', verbosity=0) + self.assertEqual(POI.objects.count(), 2) + poi = POI.objects.all().first() + self.assertEqual(poi.name, "Pic des Trois Seigneurs") + self.assertEqual(str(poi.type), 'Sommet') From 6e91b9870e09552c9826dc92d7d27fbeb7e82320 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 16:30:43 +0200 Subject: [PATCH 012/116] Add test files parser trek poi --- .../tests/data/geotrek_parser_v2/poi.json | 139 + .../data/geotrek_parser_v2/poi_type.json | 167 + .../tests/data/geotrek_parser_v2/trek.json | 3085 +++++++++++++++++ .../geotrek_parser_v2/trek_accessibility.json | 37 + .../geotrek_parser_v2/trek_difficulty.json | 62 + .../data/geotrek_parser_v2/trek_network.json | 27 + .../data/geotrek_parser_v2/trek_practice.json | 40 + .../data/geotrek_parser_v2/trek_route.json | 37 + .../data/geotrek_parser_v2/trek_theme.json | 107 + 9 files changed, 3701 insertions(+) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/poi.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json new file mode 100644 index 0000000000..b17abdcbfe --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json @@ -0,0 +1,139 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 2867, + "description": { + "fr": "

Le pic des Trois Seigneurs est situé au point de rencontre des trois vallées de la Courbière, du Vicdessos et de l'Arac (Couserans).

\r\n

Modeste sommet bien individualisé, il offre néanmoins un panorama exceptionnel sur les montagnes ariégeoises.

", + "en": "", + "es": "", + "it": "" + }, + "external_id": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.4397635, + 42.8303552, + 2166.0 + ] + }, + "name": { + "fr": "Pic des Trois Seigneurs", + "en": "", + "es": "", + "it": "" + }, + "attachments": [ + { + "backend": "", + "type": "file", + "author": "gutard", + "license": null, + "thumbnail": "", + "legend": "", + "title": "", + "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_poi/2867/dep-a5-chartreuse-bat1.pdf", + "uuid": "004c3484-d018-4706-996a-fba6e89abd96" + }, + { + "backend": "Attachment", + "type": "video", + "author": "", + "license": null, + "thumbnail": "", + "legend": "", + "title": "", + "url": "https://soundcloud.com/user-14604138/chemin-de-memoires-annie-carriere", + "uuid": "fd30f548-d870-402b-bbaf-31f895416662" + } + ], + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "type": 7, + "type_label": { + "fr": "Sommet", + "en": "Peak", + "es": null, + "it": null + }, + "type_pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-peak.png", + "url": "https://demo-admin.geotrek.fr/api/v2/poi/2867/", + "uuid": "d4f534b5-74af-40a0-9c6c-825c385b4e93", + "create_datetime": "2013-12-19T10:53:07.061717+01:00", + "update_datetime": "2020-11-24T10:52:46.696698+01:00" + }, + { + "id": 2869, + "description": { + "fr": "

Ce lac très accessible est un site prisé par les pêcheurs. On y observe truites fario, saumons des fontaines, ombles chevaliers, cristivomers et vairons.

", + "en": "", + "es": "", + "it": "" + }, + "external_id": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.4358006, + 42.8201023, + 1750.0 + ] + }, + "name": { + "fr": "Étang d'Arbu", + "en": "", + "es": "", + "it": "" + }, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "ads", + "license": null, + "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/trekking_poi/2869/sensitivearea.png.400x0_q85.png", + "legend": "", + "title": "sensitivearea", + "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_poi/2869/sensitivearea.png", + "uuid": "542afd02-487a-4df7-afaf-2082b24e8b6e" + }, + { + "backend": "Attachment", + "type": "video", + "author": "", + "license": null, + "thumbnail": "", + "legend": "", + "title": "", + "url": "https://www.youtube.com/watch?v=CdWChQqZMZc", + "uuid": "0d0542b7-b6e9-4fbd-b7c2-1438de41dfb1" + } + ], + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "type": 6, + "type_label": { + "fr": "Lac", + "en": "Lake", + "es": null, + "it": null + }, + "type_pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-lake.png", + "url": "https://demo-admin.geotrek.fr/api/v2/poi/2869/", + "uuid": "3a48fd35-654a-442d-be37-cc4c43f620bb", + "create_datetime": "2013-12-19T10:59:43.060709+01:00", + "update_datetime": "2019-04-25T11:05:26.169812+02:00" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json new file mode 100644 index 0000000000..4ce0c17c30 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json @@ -0,0 +1,167 @@ +{ + "count": 16, + "next": null, + "previous": null, + "results": [ + { + "id": 9, + "label": { + "fr": "Archéologie", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-archeology.png" + }, + { + "id": 11, + "label": { + "fr": "Architecture", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-architecture.png" + }, + { + "id": 16, + "label": { + "fr": "Barre attache", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/POIE_point_attache.png" + }, + { + "id": 15, + "label": { + "fr": "Col", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-pass.png" + }, + { + "id": 3, + "label": { + "fr": "Faune", + "en": "Fauna", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-fauna.png" + }, + { + "id": 2, + "label": { + "fr": "Flore", + "en": "Flora", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-flora.png" + }, + { + "id": 8, + "label": { + "fr": "Géologie", + "en": "Geology", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-geology.png" + }, + { + "id": 12, + "label": { + "fr": "Glacier", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-glacier.png" + }, + { + "id": 10, + "label": { + "fr": "Histoire", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-history.png" + }, + { + "id": 6, + "label": { + "fr": "Lac", + "en": "Lake", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-lake.png" + }, + { + "id": 14, + "label": { + "fr": "Pastoralisme", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-pastoral.png" + }, + { + "id": 1, + "label": { + "fr": "Petit patrimoine", + "en": "Small patrimony", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-patrimony.png" + }, + { + "id": 4, + "label": { + "fr": "Point de vue", + "en": "Panorama", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-panorama.png" + }, + { + "id": 5, + "label": { + "fr": "Refuge", + "en": "Refuge", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-refuge.png" + }, + { + "id": 13, + "label": { + "fr": "Savoir-faire", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-knowhow.png" + }, + { + "id": 7, + "label": { + "fr": "Sommet", + "en": "Peak", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-peak.png" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json new file mode 100644 index 0000000000..f0668be3ab --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -0,0 +1,3085 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 2, + "access": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [ + 1 + ], + "accessibility_advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_covering": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_exposure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_slope": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_width": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advice": { + "fr": "

Attention en cas d'orage. Fortement déconseillé par mauvais temps!

", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "Avant le Port de Lers", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://bidon.fr/api/v2/trek/2/profile/", + "ambiance": { + "fr": "

Le nom de ce pic est issu de la légende selon laquelle les trois seigneurs des vallées de Massat, Vicdessos et Rabat-les-Trois-Seigneurs, se rencontraient sur la dalle plate en son sommet afin de débattre des droits des différentes vallées qu'ils administraient.

\r\n

\r\n

À partir du xviie siècle, de grandes caravanes d'ânes et de mulets transportaient le charbon de bois entre les forêts du Couserans et les forges à la catalane de la vallée de Rabat via le col de la Pourtanelle sur l'épaulement nord du pic.

\r\n

\r\n

Au xixe siècle, des porteurs de glace venaient y chercher leur butin sur le flanc nord du pic au glacier d'Ambans pour le transporter ensuite vers Toulouse. Ce glacier a totalement disparu au début du xxie siècle.

\r\n

\r\n

\r\n

Source Wikipedia

\r\n

", + "en": "Ambiance en", + "es": "", + "it": "" + }, + "arrival": { + "fr": "Sur la route", + "en": "", + "es": "", + "it": "" + }, + "ascent": 666, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "Grégory Tonon", + "license": null, + "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg.400x0_q85.jpg", + "legend": "Vue sur la chaîne des Pyrénées, depuis le pics des 3 Seigneurs", + "title": "1024px-Les_Pyrénées_depuis_le_pic_des_3_Seigneurs", + "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg", + "uuid": "8c62ae5f-9533-4de6-a863-3e33cd42d16c" + }, + { + "backend": "Attachment", + "type": "video", + "author": "Ariège Pyrénées", + "license": null, + "thumbnail": "", + "legend": "Grands site Occitanie - collection Ariège", + "title": "", + "url": "https://www.youtube.com/watch?v=O5fwnNceuks", + "uuid": "aeffbba4-821a-4f7b-af74-e47ed628679c" + }, + { + "backend": "", + "type": "file", + "author": "", + "license": null, + "thumbnail": "", + "legend": "", + "title": "file_example_MP3_700KB", + "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/file_example_mp3_700kb.mp3", + "uuid": "529e754b-e5a8-4fe7-944d-161b4900a62e" + } + ], + "attachments_accessibility": [], + "children": [ + 10439 + ], + "cities": [ + "09231", + "09302" + ], + "create_datetime": "2013-12-12T16:20:01.405056Z", + "departure": { + "fr": "Port de Lers", + "en": "Lers bridge", + "es": "", + "it": "" + }, + "departure_city": "09231", + "departure_geom": [ + 1.411681003952174, + 42.80641572814877 + ], + "descent": -777, + "description": { + "fr": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/file_example_mp3_700kb.mp3", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "

Un très joli point de vue, en retrait de la chaîne des Pyrénées.

", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 1, + "disabled_infrastructure": { + "fr": "Accessibilité aménagement", + "en": "", + "es": "", + "it": "" + }, + "duration": 24.0, + "elevation_area_url": "https://demo-admin.geotrek.fr/api/v2/trek/2/dem/", + "elevation_svg_url": "https://demo-admin.geotrek.fr/api/v2/trek/2/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "Il faut de la corde", + "en": "", + "es": "", + "it": "" + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 1.411681, + 42.8064157, + 1516.0 + ], + [ + 1.4117038, + 42.8065905, + 1516.0 + ], + [ + 1.411729, + 42.8067836, + 1516.0 + ], + [ + 1.4117542, + 42.8069768, + 1518.0 + ], + [ + 1.4117794, + 42.80717, + 1520.0 + ], + [ + 1.4118407, + 42.8073758, + 1523.0 + ], + [ + 1.4119021, + 42.8075816, + 1528.0 + ], + [ + 1.4119634, + 42.8077874, + 1536.0 + ], + [ + 1.4120248, + 42.8079932, + 1544.0 + ], + [ + 1.4120861, + 42.808199, + 1553.0 + ], + [ + 1.412187, + 42.8083653, + 1563.0 + ], + [ + 1.4122878, + 42.8085316, + 1571.0 + ], + [ + 1.4123887, + 42.8086978, + 1578.0 + ], + [ + 1.4124895, + 42.8088641, + 1584.0 + ], + [ + 1.4125904, + 42.8090304, + 1591.0 + ], + [ + 1.4127306, + 42.8092155, + 1597.0 + ], + [ + 1.4128708, + 42.8094005, + 1603.0 + ], + [ + 1.4130111, + 42.8095856, + 1609.0 + ], + [ + 1.4131513, + 42.8097706, + 1615.0 + ], + [ + 1.4133876, + 42.8098607, + 1620.0 + ], + [ + 1.4136239, + 42.8099507, + 1625.0 + ], + [ + 1.4137589, + 42.8101307, + 1630.0 + ], + [ + 1.4138939, + 42.8103107, + 1635.0 + ], + [ + 1.4138545, + 42.810502, + 1641.0 + ], + [ + 1.4138152, + 42.8106933, + 1647.0 + ], + [ + 1.4137758, + 42.8108846, + 1652.0 + ], + [ + 1.4137364, + 42.8110759, + 1659.0 + ], + [ + 1.4137476, + 42.8112784, + 1665.0 + ], + [ + 1.4137589, + 42.8114809, + 1671.0 + ], + [ + 1.4137308, + 42.8116778, + 1679.0 + ], + [ + 1.4137027, + 42.8118747, + 1688.0 + ], + [ + 1.4136745, + 42.8120717, + 1695.0 + ], + [ + 1.4136464, + 42.8122686, + 1703.0 + ], + [ + 1.4136745, + 42.8124542, + 1712.0 + ], + [ + 1.4137026, + 42.8126399, + 1719.0 + ], + [ + 1.4137308, + 42.8128255, + 1726.0 + ], + [ + 1.4137589, + 42.8130112, + 1735.0 + ], + [ + 1.413851, + 42.8132175, + 1746.0 + ], + [ + 1.4139431, + 42.8134237, + 1755.0 + ], + [ + 1.4140352, + 42.81363, + 1766.0 + ], + [ + 1.4141274, + 42.8138362, + 1777.0 + ], + [ + 1.4142195, + 42.8140425, + 1787.0 + ], + [ + 1.4143116, + 42.8142487, + 1796.0 + ], + [ + 1.4144037, + 42.814455, + 1803.0 + ], + [ + 1.4146176, + 42.8145887, + 1811.0 + ], + [ + 1.4148315, + 42.8147224, + 1818.0 + ], + [ + 1.4150454, + 42.8148562, + 1825.0 + ], + [ + 1.4152593, + 42.8149899, + 1834.0 + ], + [ + 1.4154732, + 42.8151236, + 1844.0 + ], + [ + 1.4156871, + 42.8152573, + 1855.0 + ], + [ + 1.4158095, + 42.8154518, + 1865.0 + ], + [ + 1.4159319, + 42.8156464, + 1875.0 + ], + [ + 1.4160543, + 42.8158409, + 1887.0 + ], + [ + 1.4161767, + 42.8160354, + 1896.0 + ], + [ + 1.4161243, + 42.8162275, + 1905.0 + ], + [ + 1.416072, + 42.8164195, + 1912.0 + ], + [ + 1.4160196, + 42.8166116, + 1918.0 + ], + [ + 1.4158625, + 42.8167351, + 1921.0 + ], + [ + 1.4157054, + 42.8168586, + 1923.0 + ], + [ + 1.4155483, + 42.816982, + 1922.0 + ], + [ + 1.4153912, + 42.8171055, + 1923.0 + ], + [ + 1.415288, + 42.8172932, + 1926.0 + ], + [ + 1.4151847, + 42.8174808, + 1929.0 + ], + [ + 1.4150815, + 42.8176685, + 1934.0 + ], + [ + 1.4149782, + 42.8178561, + 1939.0 + ], + [ + 1.414875, + 42.8180438, + 1941.0 + ], + [ + 1.4150321, + 42.8182149, + 1942.0 + ], + [ + 1.4151892, + 42.818386, + 1941.0 + ], + [ + 1.4153464, + 42.8185572, + 1940.0 + ], + [ + 1.4155035, + 42.8187283, + 1940.0 + ], + [ + 1.4156606, + 42.8188994, + 1942.0 + ], + [ + 1.4158139, + 42.8190861, + 1943.0 + ], + [ + 1.4159673, + 42.8192727, + 1944.0 + ], + [ + 1.4161206, + 42.8194594, + 1945.0 + ], + [ + 1.4162739, + 42.819646, + 1946.0 + ], + [ + 1.4164272, + 42.8198327, + 1947.0 + ], + [ + 1.4165806, + 42.8200193, + 1952.0 + ], + [ + 1.4167339, + 42.820206, + 1957.0 + ], + [ + 1.4169439, + 42.8203343, + 1963.0 + ], + [ + 1.4171539, + 42.8204627, + 1969.0 + ], + [ + 1.4173639, + 42.820591, + 1977.0 + ], + [ + 1.4175739, + 42.8207193, + 1981.0 + ], + [ + 1.4177255, + 42.820906, + 1983.0 + ], + [ + 1.4178772, + 42.8210926, + 1984.0 + ], + [ + 1.4180288, + 42.8212793, + 1982.0 + ], + [ + 1.4181805, + 42.8214659, + 1979.0 + ], + [ + 1.4183205, + 42.8216246, + 1976.0 + ], + [ + 1.4184605, + 42.8217832, + 1973.0 + ], + [ + 1.4186005, + 42.8219419, + 1970.0 + ], + [ + 1.4187405, + 42.8221005, + 1968.0 + ], + [ + 1.4188805, + 42.8222592, + 1965.0 + ], + [ + 1.4189738, + 42.8224459, + 1963.0 + ], + [ + 1.4190671, + 42.8226325, + 1961.0 + ], + [ + 1.4191604, + 42.8228192, + 1960.0 + ], + [ + 1.4192538, + 42.8230059, + 1959.0 + ], + [ + 1.4193471, + 42.8231925, + 1959.0 + ], + [ + 1.4194404, + 42.8233792, + 1960.0 + ], + [ + 1.4195937, + 42.8235459, + 1961.0 + ], + [ + 1.4197471, + 42.8237125, + 1962.0 + ], + [ + 1.4199004, + 42.8238792, + 1963.0 + ], + [ + 1.4200537, + 42.8240458, + 1965.0 + ], + [ + 1.420207, + 42.8242125, + 1966.0 + ], + [ + 1.4203604, + 42.8243791, + 1969.0 + ], + [ + 1.4205137, + 42.8245458, + 1973.0 + ], + [ + 1.4207678, + 42.8246288, + 1977.0 + ], + [ + 1.4210218, + 42.8247117, + 1981.0 + ], + [ + 1.4212759, + 42.8247947, + 1985.0 + ], + [ + 1.42153, + 42.8248776, + 1985.0 + ], + [ + 1.421784, + 42.8249606, + 1985.0 + ], + [ + 1.4220381, + 42.8250435, + 1985.0 + ], + [ + 1.4222922, + 42.8251265, + 1986.0 + ], + [ + 1.4225462, + 42.8252094, + 1989.0 + ], + [ + 1.4228003, + 42.8252924, + 1992.0 + ], + [ + 1.423047, + 42.8253924, + 1998.0 + ], + [ + 1.4232936, + 42.8254924, + 2003.0 + ], + [ + 1.4235403, + 42.8255924, + 2005.0 + ], + [ + 1.4237869, + 42.8256924, + 2006.0 + ], + [ + 1.4240336, + 42.8257924, + 2008.0 + ], + [ + 1.4242802, + 42.8258924, + 2009.0 + ], + [ + 1.4245269, + 42.8259924, + 2012.0 + ], + [ + 1.4248011, + 42.8260844, + 2018.0 + ], + [ + 1.4250753, + 42.8261765, + 2022.0 + ], + [ + 1.4253494, + 42.8262685, + 2023.0 + ], + [ + 1.4256236, + 42.8263606, + 2021.0 + ], + [ + 1.4258978, + 42.8264526, + 2019.0 + ], + [ + 1.426172, + 42.8265446, + 2016.0 + ], + [ + 1.4264462, + 42.8266367, + 2013.0 + ], + [ + 1.4267203, + 42.8267287, + 2013.0 + ], + [ + 1.4269945, + 42.8268208, + 2015.0 + ], + [ + 1.4272687, + 42.8269128, + 2016.0 + ], + [ + 1.4275282, + 42.827026, + 2017.0 + ], + [ + 1.4277877, + 42.8271393, + 2018.0 + ], + [ + 1.4280472, + 42.8272525, + 2018.0 + ], + [ + 1.4283067, + 42.8273658, + 2018.0 + ], + [ + 1.4285662, + 42.827479, + 2018.0 + ], + [ + 1.4288257, + 42.8275923, + 2019.0 + ], + [ + 1.4290852, + 42.8277055, + 2019.0 + ], + [ + 1.429323, + 42.8278244, + 2019.0 + ], + [ + 1.4295608, + 42.8279433, + 2019.0 + ], + [ + 1.4297986, + 42.8280622, + 2019.0 + ], + [ + 1.4300364, + 42.8281811, + 2018.0 + ], + [ + 1.4302742, + 42.8283, + 2016.0 + ], + [ + 1.430512, + 42.8284189, + 2014.0 + ], + [ + 1.4307499, + 42.8285378, + 2013.0 + ], + [ + 1.4309877, + 42.8286567, + 2013.0 + ], + [ + 1.4312255, + 42.8287756, + 2016.0 + ], + [ + 1.4314633, + 42.8288945, + 2021.0 + ], + [ + 1.431711, + 42.8289441, + 2028.0 + ], + [ + 1.4319587, + 42.8289936, + 2033.0 + ], + [ + 1.4321073, + 42.8289606, + 2038.0 + ], + [ + 1.4322559, + 42.8289275, + 2041.0 + ], + [ + 1.4324871, + 42.8289606, + 2042.0 + ], + [ + 1.4327182, + 42.8289936, + 2041.0 + ], + [ + 1.4327183, + 42.8289936, + 2041.0 + ], + [ + 1.4329385, + 42.8290266, + 2040.0 + ], + [ + 1.4331587, + 42.8290596, + 2037.0 + ], + [ + 1.4333789, + 42.8290926, + 2035.0 + ], + [ + 1.4335683, + 42.8291781, + 2031.0 + ], + [ + 1.4337577, + 42.8292636, + 2026.0 + ], + [ + 1.4340382, + 42.8293241, + 2020.0 + ], + [ + 1.4339593, + 42.8291741, + 2015.0 + ], + [ + 1.4339948, + 42.8290636, + 2009.0 + ], + [ + 1.4340303, + 42.828953, + 2002.0 + ], + [ + 1.4342435, + 42.8287952, + 1996.0 + ], + [ + 1.4342988, + 42.8286729, + 1991.0 + ], + [ + 1.434354, + 42.8285505, + 1987.0 + ], + [ + 1.4343974, + 42.8283649, + 1984.0 + ], + [ + 1.4344408, + 42.8281794, + 1982.0 + ], + [ + 1.4345277, + 42.8280216, + 1982.0 + ], + [ + 1.4346422, + 42.8279387, + 1982.0 + ], + [ + 1.4347566, + 42.8278558, + 1982.0 + ], + [ + 1.4347568, + 42.8278555, + 1981.0 + ], + [ + 1.4348513, + 42.8277295, + 1980.0 + ], + [ + 1.43486, + 42.8275735, + 1978.0 + ], + [ + 1.4348686, + 42.8274176, + 1975.0 + ], + [ + 1.4348934, + 42.8272506, + 1972.0 + ], + [ + 1.4349181, + 42.8270837, + 1968.0 + ], + [ + 1.4350047, + 42.8269662, + 1963.0 + ], + [ + 1.4350912, + 42.8268487, + 1958.0 + ], + [ + 1.4352891, + 42.8267683, + 1953.0 + ], + [ + 1.435487, + 42.8266879, + 1949.0 + ], + [ + 1.4355942, + 42.8265189, + 1945.0 + ], + [ + 1.4357014, + 42.8263498, + 1943.0 + ], + [ + 1.4358086, + 42.8261808, + 1940.0 + ], + [ + 1.435905, + 42.8259814, + 1937.0 + ], + [ + 1.4360013, + 42.8257819, + 1933.0 + ], + [ + 1.4360337, + 42.8255716, + 1927.0 + ], + [ + 1.4359447, + 42.8253855, + 1919.0 + ], + [ + 1.4358557, + 42.825244, + 1912.0 + ], + [ + 1.4357667, + 42.8251024, + 1905.0 + ], + [ + 1.4356964, + 42.8249714, + 1898.0 + ], + [ + 1.4356261, + 42.8248404, + 1892.0 + ], + [ + 1.4354401, + 42.8247067, + 1890.0 + ], + [ + 1.4352541, + 42.824573, + 1887.0 + ], + [ + 1.435068, + 42.8244392, + 1884.0 + ], + [ + 1.434882, + 42.8243055, + 1880.0 + ], + [ + 1.434696, + 42.8241718, + 1877.0 + ], + [ + 1.4345507, + 42.8240207, + 1870.0 + ], + [ + 1.4344053, + 42.8238695, + 1863.0 + ], + [ + 1.43426, + 42.8237184, + 1856.0 + ], + [ + 1.4341146, + 42.8235672, + 1848.0 + ], + [ + 1.4339693, + 42.8234161, + 1840.0 + ], + [ + 1.4338046, + 42.8233192, + 1833.0 + ], + [ + 1.4336398, + 42.8232223, + 1826.0 + ], + [ + 1.4334751, + 42.8231254, + 1821.0 + ], + [ + 1.4334557, + 42.822951, + 1816.0 + ], + [ + 1.4334364, + 42.8227766, + 1813.0 + ], + [ + 1.433417, + 42.8226022, + 1809.0 + ], + [ + 1.433417, + 42.8224181, + 1804.0 + ], + [ + 1.433417, + 42.822234, + 1798.0 + ], + [ + 1.433417, + 42.8220499, + 1791.0 + ], + [ + 1.4336011, + 42.8219142, + 1784.0 + ], + [ + 1.4337852, + 42.8217786, + 1776.0 + ], + [ + 1.4339693, + 42.8216429, + 1770.0 + ], + [ + 1.4341728, + 42.8215092, + 1765.0 + ], + [ + 1.4343762, + 42.8213755, + 1760.0 + ], + [ + 1.4345797, + 42.8212417, + 1757.0 + ], + [ + 1.4347831, + 42.821108, + 1754.0 + ], + [ + 1.4349866, + 42.8209743, + 1753.0 + ], + [ + 1.4351494, + 42.8207999, + 1752.0 + ], + [ + 1.4353122, + 42.8206255, + 1752.0 + ], + [ + 1.435475, + 42.8204511, + 1752.0 + ], + [ + 1.4356378, + 42.8202767, + 1752.0 + ], + [ + 1.4358006, + 42.8201023, + 1751.0 + ], + [ + 1.4359895, + 42.819986, + 1750.0 + ], + [ + 1.4361785, + 42.8198698, + 1749.0 + ], + [ + 1.4363674, + 42.8197535, + 1746.0 + ], + [ + 1.4365563, + 42.8196372, + 1742.0 + ], + [ + 1.4366871, + 42.8195064, + 1738.0 + ], + [ + 1.4368179, + 42.8193756, + 1733.0 + ], + [ + 1.4370824, + 42.8193016, + 1730.0 + ], + [ + 1.4373469, + 42.8192275, + 1727.0 + ], + [ + 1.4376114, + 42.8191535, + 1726.0 + ], + [ + 1.4378396, + 42.8191193, + 1724.0 + ], + [ + 1.4380677, + 42.819085, + 1723.0 + ], + [ + 1.4380677, + 42.8189481, + 1720.0 + ], + [ + 1.4380677, + 42.8188112, + 1717.0 + ], + [ + 1.4381019, + 42.8186287, + 1713.0 + ], + [ + 1.4381361, + 42.8184462, + 1709.0 + ], + [ + 1.4381475, + 42.8182294, + 1706.0 + ], + [ + 1.4381589, + 42.8180127, + 1704.0 + ], + [ + 1.4382046, + 42.8178644, + 1704.0 + ], + [ + 1.4382502, + 42.8177161, + 1705.0 + ], + [ + 1.4382616, + 42.8175335, + 1706.0 + ], + [ + 1.438273, + 42.817351, + 1707.0 + ], + [ + 1.4382046, + 42.8171457, + 1706.0 + ], + [ + 1.4381361, + 42.8169403, + 1703.0 + ], + [ + 1.4380677, + 42.816735, + 1698.0 + ], + [ + 1.4380312, + 42.816557, + 1693.0 + ], + [ + 1.4379947, + 42.8163791, + 1687.0 + ], + [ + 1.4379582, + 42.8162011, + 1680.0 + ], + [ + 1.4379217, + 42.8160232, + 1672.0 + ], + [ + 1.4378852, + 42.8158452, + 1666.0 + ], + [ + 1.4378681, + 42.8156398, + 1660.0 + ], + [ + 1.4378509, + 42.8154345, + 1653.0 + ], + [ + 1.4378338, + 42.8152291, + 1647.0 + ], + [ + 1.4378167, + 42.8150238, + 1641.0 + ], + [ + 1.4378015, + 42.8148337, + 1636.0 + ], + [ + 1.4377863, + 42.8146435, + 1630.0 + ], + [ + 1.4377711, + 42.8144534, + 1624.0 + ], + [ + 1.4377863, + 42.8142785, + 1618.0 + ], + [ + 1.4378015, + 42.8141035, + 1612.0 + ], + [ + 1.4378167, + 42.8139286, + 1605.0 + ], + [ + 1.4378738, + 42.813746, + 1600.0 + ], + [ + 1.4379308, + 42.8135635, + 1596.0 + ], + [ + 1.4380563, + 42.8134951, + 1592.0 + ], + [ + 1.4381818, + 42.8134266, + 1588.0 + ], + [ + 1.438022, + 42.8132669, + 1582.0 + ], + [ + 1.4378623, + 42.8131072, + 1574.0 + ], + [ + 1.4376969, + 42.8129874, + 1564.0 + ], + [ + 1.4375315, + 42.8128677, + 1552.0 + ], + [ + 1.4373661, + 42.8127479, + 1540.0 + ], + [ + 1.4372007, + 42.8126281, + 1528.0 + ], + [ + 1.4370258, + 42.8125216, + 1517.0 + ], + [ + 1.4368508, + 42.8124152, + 1507.0 + ], + [ + 1.4366759, + 42.8123087, + 1497.0 + ], + [ + 1.4364788, + 42.8122185, + 1488.0 + ], + [ + 1.4362818, + 42.8121283, + 1480.0 + ], + [ + 1.4360659, + 42.8122342, + 1473.0 + ], + [ + 1.4358459, + 42.8121406, + 1466.0 + ], + [ + 1.435626, + 42.8120469, + 1460.0 + ], + [ + 1.4354631, + 42.8121542, + 1455.0 + ], + [ + 1.4353001, + 42.8122614, + 1450.0 + ], + [ + 1.4351372, + 42.8123687, + 1445.0 + ], + [ + 1.4348774, + 42.8123797, + 1441.0 + ], + [ + 1.4346176, + 42.8123907, + 1437.0 + ], + [ + 1.4343578, + 42.8124017, + 1435.0 + ], + [ + 1.4342931, + 42.8123409, + 1433.0 + ], + [ + 1.4343238, + 42.8122299, + 1431.0 + ], + [ + 1.4343544, + 42.812119, + 1430.0 + ], + [ + 1.4343131, + 42.8120769, + 1428.0 + ], + [ + 1.434085, + 42.8120582, + 1426.0 + ], + [ + 1.4339295, + 42.81192, + 1423.0 + ], + [ + 1.4337741, + 42.8117817, + 1421.0 + ], + [ + 1.4335812, + 42.8116734, + 1419.0 + ], + [ + 1.4333474, + 42.8116359, + 1418.0 + ], + [ + 1.4332695, + 42.8116497, + 1416.0 + ], + [ + 1.4331915, + 42.8116852, + 1415.0 + ], + [ + 1.4330491, + 42.8115945, + 1414.0 + ], + [ + 1.4328631, + 42.8115593, + 1412.0 + ], + [ + 1.4326771, + 42.8115241, + 1409.0 + ], + [ + 1.4325105, + 42.8114348, + 1407.0 + ], + [ + 1.4323439, + 42.8113455, + 1403.0 + ], + [ + 1.4321773, + 42.8112562, + 1400.0 + ], + [ + 1.4319525, + 42.8111998, + 1397.0 + ], + [ + 1.4317277, + 42.8111435, + 1396.0 + ], + [ + 1.431503, + 42.8110871, + 1395.0 + ], + [ + 1.4312782, + 42.8110307, + 1395.0 + ], + [ + 1.4310659, + 42.8110366, + 1396.0 + ], + [ + 1.4309987, + 42.8109893, + 1397.0 + ], + [ + 1.4307793, + 42.8109726, + 1397.0 + ], + [ + 1.43056, + 42.8109559, + 1398.0 + ], + [ + 1.4304317, + 42.8109735, + 1399.0 + ], + [ + 1.4301818, + 42.8109045, + 1401.0 + ], + [ + 1.4300514, + 42.8108415, + 1403.0 + ], + [ + 1.4299211, + 42.8107784, + 1404.0 + ], + [ + 1.4296469, + 42.8107393, + 1405.0 + ], + [ + 1.4293728, + 42.8107001, + 1407.0 + ], + [ + 1.4291573, + 42.8106532, + 1408.0 + ], + [ + 1.4289417, + 42.8106064, + 1409.0 + ], + [ + 1.4287262, + 42.8105595, + 1410.0 + ], + [ + 1.4285199, + 42.8104738, + 1413.0 + ], + [ + 1.4283137, + 42.8103881, + 1415.0 + ], + [ + 1.4281968, + 42.8103029, + 1417.0 + ], + [ + 1.42808, + 42.8102177, + 1418.0 + ], + [ + 1.4279303, + 42.810098, + 1420.0 + ], + [ + 1.4278505, + 42.8098984, + 1420.0 + ], + [ + 1.4279127, + 42.8096897, + 1421.0 + ], + [ + 1.4279593, + 42.8095169, + 1422.0 + ], + [ + 1.4280058, + 42.8093441, + 1422.0 + ], + [ + 1.4280581, + 42.8092206, + 1423.0 + ], + [ + 1.4281104, + 42.809097, + 1422.0 + ], + [ + 1.4281494, + 42.8089847, + 1421.0 + ], + [ + 1.4281883, + 42.8088723, + 1419.0 + ], + [ + 1.428204, + 42.808672, + 1416.0 + ], + [ + 1.4282196, + 42.8084718, + 1411.0 + ], + [ + 1.4282381, + 42.808257, + 1405.0 + ], + [ + 1.4281994, + 42.8080872, + 1398.0 + ], + [ + 1.4281607, + 42.8079174, + 1391.0 + ], + [ + 1.4279773, + 42.8077578, + 1382.0 + ], + [ + 1.4277756, + 42.8076956, + 1375.0 + ], + [ + 1.427574, + 42.8076334, + 1369.0 + ], + [ + 1.4272872, + 42.8076402, + 1364.0 + ], + [ + 1.4270004, + 42.807647, + 1360.0 + ], + [ + 1.4267136, + 42.8076538, + 1358.0 + ], + [ + 1.4264724, + 42.807701, + 1358.0 + ], + [ + 1.4262313, + 42.8077482, + 1360.0 + ], + [ + 1.4259901, + 42.8077954, + 1364.0 + ], + [ + 1.4257177, + 42.8078459, + 1370.0 + ], + [ + 1.4254454, + 42.8078965, + 1376.0 + ], + [ + 1.425173, + 42.807947, + 1380.0 + ], + [ + 1.4250661, + 42.8079604, + 1383.0 + ], + [ + 1.4249892, + 42.8079537, + 1385.0 + ], + [ + 1.4248756, + 42.8079671, + 1386.0 + ], + [ + 1.4247887, + 42.8080072, + 1387.0 + ], + [ + 1.4247253, + 42.808074, + 1387.0 + ], + [ + 1.4247012, + 42.8081018, + 1388.0 + ], + [ + 1.4246277, + 42.8080722, + 1386.0 + ], + [ + 1.4245895, + 42.8079349, + 1383.0 + ], + [ + 1.4245509, + 42.8079029, + 1380.0 + ], + [ + 1.4244103, + 42.8079007, + 1376.0 + ], + [ + 1.4242745, + 42.807814, + 1372.0 + ], + [ + 1.4241388, + 42.8077273, + 1370.0 + ], + [ + 1.4240101, + 42.8075935, + 1371.0 + ], + [ + 1.4238485, + 42.8075043, + 1373.0 + ], + [ + 1.423687, + 42.8074151, + 1375.0 + ], + [ + 1.4234755, + 42.8073151, + 1377.0 + ], + [ + 1.423264, + 42.8072151, + 1378.0 + ], + [ + 1.4230455, + 42.807167, + 1378.0 + ], + [ + 1.4228271, + 42.8071189, + 1378.0 + ], + [ + 1.4226651, + 42.8071154, + 1378.0 + ], + [ + 1.4225032, + 42.8071118, + 1380.0 + ], + [ + 1.4222446, + 42.8070572, + 1383.0 + ], + [ + 1.4219817, + 42.8070129, + 1386.0 + ], + [ + 1.4218847, + 42.806984, + 1389.0 + ], + [ + 1.4218024, + 42.8069175, + 1392.0 + ], + [ + 1.4215877, + 42.8068009, + 1395.0 + ], + [ + 1.4214108, + 42.8066672, + 1398.0 + ], + [ + 1.4212519, + 42.8066211, + 1402.0 + ], + [ + 1.421093, + 42.8065749, + 1403.0 + ], + [ + 1.4209135, + 42.8065618, + 1405.0 + ] + ] + }, + "gpx": "https://demo-admin.geotrek.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.gpx", + "information_desks": [ + 2, + 1, + 3 + ], + "kml": "https://demo-admin.geotrek.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.kml", + "labels": [ + 1, + 2, + 3 + ], + "length_2d": 7606.3, + "length_3d": 7811.9, + "max_elevation": 2042, + "min_elevation": 1358, + "name": { + "fr": "Boucle du Pic des Trois Seigneurs", + "en": "Loop of the pic of 3 lords", + "es": "", + "it": "" + }, + "networks": [ + 2 + ], + "next": {}, + "parents": [], + "parking_location": [ + 1.412816, + 42.8063269 + ], + "pdf": { + "fr": "https://demo-admin.geotrek.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "en": "https://demo-admin.geotrek.fr/api/en/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "es": "https://demo-admin.geotrek.fr/api/es/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "it": "https://demo-admin.geotrek.fr/api/it/treks/2/boucle-du-pic-des-trois-seigneurs.pdf" + }, + "points_reference": { + "type": "MultiPoint", + "coordinates": [ + [ + 1.411417008494028, + 42.81070319713575 + ], + [ + 1.415279389475474, + 42.81693648180148 + ], + [ + 1.403348923777236, + 42.825246550724195 + ], + [ + 1.428325654123916, + 42.8286458024494 + ], + [ + 1.437852860544806, + 42.81611800550653 + ], + [ + 1.423759460449225, + 42.807239989743145 + ] + ] + }, + "portal": [], + "practice": 3, + "ratings": [], + "ratings_description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "previous": {}, + "public_transport": { + "fr": "Test", + "en": "", + "es": "", + "it": "" + }, + "published": { + "fr": true, + "en": true, + "es": true, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 1, + "second_external_id": null, + "source": [], + "structure": 1, + "themes": [ + 8, + 4 + ], + "update_datetime": "2022-05-16T12:10:59.927409Z", + "url": "https://demo-admin.geotrek.fr/api/v2/trek/2/", + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1", + "web_links": [ + { + "name": { + "fr": "Camping", + "en": "Camping", + "es": "Camping", + "it": null + }, + "url": "http://camping.com/", + "category": { + "label": { + "fr": "A lire", + "en": null, + "es": null, + "it": null + }, + "id": 4, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/weblink-book.png" + } + }, + { + "name": { + "fr": "Makina Corpus", + "en": "Makina Corpus", + "es": "Makina Corpus", + "it": null + }, + "url": "http://makina-corpus.com/", + "category": { + "label": { + "fr": "A lire", + "en": null, + "es": null, + "it": null + }, + "id": 4, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/weblink-book.png" + } + }, + { + "name": { + "fr": "Test", + "en": "", + "es": "", + "it": "" + }, + "url": "http://toto.com", + "category": { + "label": { + "fr": "A lire", + "en": null, + "es": null, + "it": null + }, + "id": 4, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/weblink-book.png" + } + } + ] + }, + { + "id": 10443, + "access": { + "fr": "Bonjour", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [], + "accessibility_advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_covering": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_exposure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_slope": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_width": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://demo-admin.geotrek.fr/api/v2/trek/10443/profile/", + "ambiance": { + "fr": "Test ambiance", + "en": "", + "es": "", + "it": "" + }, + "arrival": { + "fr": "col de la Core", + "en": "", + "es": "", + "it": "" + }, + "ascent": 722, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "Samrong01 - CC BY-SA 4.0", + "license": null, + "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/10443/1083px-arrien-en-bethmale_general_view.JPG.400x0_q85.jpg", + "legend": "Arrien-en-Bethmale, vue du village", + "title": "", + "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/10443/1083px-arrien-en-bethmale_general_view.JPG", + "uuid": "8605f89a-48fa-40ba-a9b0-9a48e5b0f310" + } + ], + "attachments_accessibility": [], + "children": [], + "cities": [ + "09055", + "09291" + ], + "create_datetime": "2019-07-23T09:15:06.776780Z", + "departure": { + "fr": "Bethmale", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09055", + "departure_geom": [ + 1.0853185207621108, + 42.86403321156401 + ], + "descent": -69, + "description": { + "fr": "La belle description de la crête.", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "Au revoir", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 2, + "disabled_infrastructure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "duration": 2.0, + "elevation_area_url": "https://demo-admin.geotrek.fr/api/v2/trek/10443/dem/", + "elevation_svg_url": "https://demo-admin.geotrek.fr/api/v2/trek/10443/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 1.0853185, + 42.8640332, + 1057.0 + ], + [ + 1.0852891, + 42.8640109, + 1061.0 + ], + [ + 1.0855466, + 42.8640926, + 1063.0 + ], + [ + 1.085804, + 42.8641744, + 1067.0 + ], + [ + 1.0860615, + 42.8642562, + 1072.0 + ], + [ + 1.086319, + 42.864338, + 1075.0 + ], + [ + 1.0865765, + 42.8644198, + 1076.0 + ], + [ + 1.086834, + 42.8645016, + 1079.0 + ], + [ + 1.0870915, + 42.8645833, + 1082.0 + ], + [ + 1.087349, + 42.8646651, + 1084.0 + ], + [ + 1.0876065, + 42.8647469, + 1086.0 + ], + [ + 1.087864, + 42.8648287, + 1088.0 + ], + [ + 1.0881215, + 42.8647186, + 1090.0 + ], + [ + 1.088379, + 42.8646085, + 1092.0 + ], + [ + 1.0886365, + 42.8644984, + 1094.0 + ], + [ + 1.088894, + 42.8643883, + 1096.0 + ], + [ + 1.0891515, + 42.8642782, + 1097.0 + ], + [ + 1.089409, + 42.8641681, + 1097.0 + ], + [ + 1.0896665, + 42.864058, + 1096.0 + ], + [ + 1.089924, + 42.863948, + 1094.0 + ], + [ + 1.0901814, + 42.8638379, + 1094.0 + ], + [ + 1.0904389, + 42.8637278, + 1096.0 + ], + [ + 1.0906964, + 42.8636177, + 1096.0 + ], + [ + 1.0909539, + 42.8635076, + 1098.0 + ], + [ + 1.0912114, + 42.8633975, + 1100.0 + ], + [ + 1.0914689, + 42.8632874, + 1103.0 + ], + [ + 1.0917264, + 42.8631773, + 1106.0 + ], + [ + 1.0919839, + 42.8630672, + 1110.0 + ], + [ + 1.0922168, + 42.8629324, + 1115.0 + ], + [ + 1.0924498, + 42.8627976, + 1119.0 + ], + [ + 1.0926828, + 42.8626627, + 1123.0 + ], + [ + 1.0929157, + 42.8625279, + 1127.0 + ], + [ + 1.0931487, + 42.8623931, + 1132.0 + ], + [ + 1.0933817, + 42.8622583, + 1136.0 + ], + [ + 1.0936146, + 42.8621235, + 1139.0 + ], + [ + 1.0937005, + 42.8619168, + 1145.0 + ], + [ + 1.0937863, + 42.86171, + 1152.0 + ], + [ + 1.0938721, + 42.8615033, + 1157.0 + ], + [ + 1.093958, + 42.8612966, + 1162.0 + ], + [ + 1.0940438, + 42.8610899, + 1167.0 + ], + [ + 1.0941296, + 42.8608832, + 1169.0 + ], + [ + 1.0942155, + 42.8606765, + 1169.0 + ], + [ + 1.0941255, + 42.8604667, + 1167.0 + ], + [ + 1.0940356, + 42.860257, + 1164.0 + ], + [ + 1.0939457, + 42.8600473, + 1161.0 + ], + [ + 1.0938558, + 42.8598376, + 1158.0 + ], + [ + 1.0937658, + 42.8596278, + 1155.0 + ], + [ + 1.0936759, + 42.8594181, + 1155.0 + ], + [ + 1.093586, + 42.8592084, + 1155.0 + ], + [ + 1.0934961, + 42.8589987, + 1157.0 + ], + [ + 1.0934062, + 42.858789, + 1157.0 + ], + [ + 1.0933162, + 42.8585792, + 1157.0 + ], + [ + 1.0932263, + 42.8583695, + 1154.0 + ], + [ + 1.0931364, + 42.8581598, + 1151.0 + ], + [ + 1.0930465, + 42.8579501, + 1147.0 + ], + [ + 1.0929566, + 42.8577403, + 1145.0 + ], + [ + 1.0928667, + 42.8575306, + 1144.0 + ], + [ + 1.0927767, + 42.8573209, + 1145.0 + ], + [ + 1.0926868, + 42.8571112, + 1148.0 + ], + [ + 1.0925969, + 42.8569014, + 1152.0 + ], + [ + 1.092507, + 42.8566917, + 1156.0 + ], + [ + 1.0924171, + 42.856482, + 1159.0 + ], + [ + 1.0923272, + 42.8562723, + 1161.0 + ], + [ + 1.0924676, + 42.8560892, + 1162.0 + ], + [ + 1.0926081, + 42.8559062, + 1163.0 + ], + [ + 1.0927485, + 42.8557232, + 1164.0 + ], + [ + 1.092889, + 42.8555401, + 1167.0 + ], + [ + 1.0930294, + 42.8553571, + 1171.0 + ], + [ + 1.0931699, + 42.855174, + 1176.0 + ], + [ + 1.0933103, + 42.854991, + 1183.0 + ], + [ + 1.0934508, + 42.8548079, + 1191.0 + ], + [ + 1.0935912, + 42.8546249, + 1197.0 + ], + [ + 1.0937317, + 42.8544419, + 1201.0 + ], + [ + 1.0938721, + 42.8542588, + 1205.0 + ], + [ + 1.0941094, + 42.854381, + 1202.0 + ], + [ + 1.0943467, + 42.8545031, + 1195.0 + ], + [ + 1.094584, + 42.8546253, + 1188.0 + ], + [ + 1.0948213, + 42.8547474, + 1181.0 + ], + [ + 1.0950586, + 42.8548695, + 1174.0 + ], + [ + 1.0952959, + 42.8549917, + 1169.0 + ], + [ + 1.0955332, + 42.8551138, + 1167.0 + ], + [ + 1.0957705, + 42.855236, + 1167.0 + ], + [ + 1.0960078, + 42.8553581, + 1167.0 + ], + [ + 1.0962451, + 42.8554802, + 1168.0 + ], + [ + 1.0964824, + 42.8556024, + 1172.0 + ], + [ + 1.0967197, + 42.8557245, + 1175.0 + ], + [ + 1.096957, + 42.8558466, + 1179.0 + ], + [ + 1.0971943, + 42.8559688, + 1185.0 + ], + [ + 1.0974316, + 42.8560909, + 1192.0 + ], + [ + 1.0976689, + 42.8562131, + 1201.0 + ], + [ + 1.0979062, + 42.8563352, + 1212.0 + ], + [ + 1.098044, + 42.8561365, + 1225.0 + ], + [ + 1.0981818, + 42.8559378, + 1240.0 + ], + [ + 1.0983196, + 42.8557391, + 1258.0 + ], + [ + 1.0984574, + 42.8555404, + 1273.0 + ], + [ + 1.0985952, + 42.8553417, + 1286.0 + ], + [ + 1.098733, + 42.855143, + 1300.0 + ], + [ + 1.0988707, + 42.8549443, + 1313.0 + ], + [ + 1.0990085, + 42.8547456, + 1325.0 + ], + [ + 1.0991463, + 42.8545469, + 1340.0 + ], + [ + 1.0992841, + 42.8543482, + 1359.0 + ], + [ + 1.0994219, + 42.8541495, + 1374.0 + ], + [ + 1.0995597, + 42.8539508, + 1388.0 + ], + [ + 1.0996975, + 42.8537521, + 1404.0 + ], + [ + 1.0998353, + 42.8535534, + 1418.0 + ], + [ + 1.0999731, + 42.8533547, + 1432.0 + ], + [ + 1.1001108, + 42.8531559, + 1449.0 + ], + [ + 1.1002486, + 42.8529572, + 1468.0 + ], + [ + 1.1003864, + 42.8527585, + 1485.0 + ], + [ + 1.1005242, + 42.8525598, + 1501.0 + ], + [ + 1.100662, + 42.8523611, + 1521.0 + ], + [ + 1.1007997, + 42.8521624, + 1541.0 + ], + [ + 1.1009375, + 42.8519637, + 1562.0 + ], + [ + 1.1010753, + 42.851765, + 1588.0 + ], + [ + 1.1012131, + 42.8515663, + 1615.0 + ], + [ + 1.1013509, + 42.8513676, + 1637.0 + ], + [ + 1.1014886, + 42.8511689, + 1656.0 + ], + [ + 1.1016264, + 42.8509702, + 1672.0 + ], + [ + 1.1017642, + 42.8507715, + 1683.0 + ], + [ + 1.1019019, + 42.8505728, + 1690.0 + ], + [ + 1.1020397, + 42.8503741, + 1696.0 + ], + [ + 1.1021775, + 42.8501754, + 1700.0 + ], + [ + 1.1023153, + 42.8499767, + 1704.0 + ], + [ + 1.102453, + 42.849778, + 1707.0 + ], + [ + 1.1025908, + 42.8495793, + 1709.0 + ], + [ + 1.1027286, + 42.8493805, + 1710.0 + ], + [ + 1.1028663, + 42.8491818, + 1711.0 + ], + [ + 1.1030041, + 42.8489831, + 1711.0 + ], + [ + 1.1030671, + 42.8488923, + 1710.0 + ] + ] + }, + "gpx": "https://demo-admin.geotrek.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.gpx", + "information_desks": [], + "kml": "https://demo-admin.geotrek.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.kml", + "labels": [], + "length_2d": 3063.4, + "length_3d": 3245.4, + "max_elevation": 1711, + "min_elevation": 1057, + "name": { + "fr": "De Bethmale au col de la Core", + "en": "", + "es": "", + "it": "" + }, + "networks": [], + "next": { + "10445": null + }, + "parents": [ + 10445 + ], + "parking_location": null, + "pdf": { + "fr": "https://demo-admin.geotrek.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "en": "https://demo-admin.geotrek.fr/api/en/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "es": "https://demo-admin.geotrek.fr/api/es/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "it": "https://demo-admin.geotrek.fr/api/it/treks/10443/de-bethmale-au-col-de-la-core.pdf" + }, + "points_reference": null, + "portal": [], + "practice": 4, + "ratings": [], + "ratings_description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "previous": { + "10445": 10441 + }, + "public_transport": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 3, + "second_external_id": null, + "source": [], + "structure": 3, + "themes": [], + "update_datetime": "2022-07-26T08:18:06.997582Z", + "url": "https://demo-admin.geotrek.fr/api/v2/trek/10443/", + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12", + "web_links": [] + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json new file mode 100644 index 0000000000..17fe26726c --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json @@ -0,0 +1,37 @@ +{ + "count": 3, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "name": { + "fr": "Fauteuil roulant", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/accessibility-wheelchair.png" + }, + { + "id": 2, + "name": { + "fr": "Poussette", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/accessibility-troller.png" + }, + { + "id": 3, + "name": { + "fr": "Joelette", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/accessibility-joelette.png" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json new file mode 100644 index 0000000000..b8f3236290 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json @@ -0,0 +1,62 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "cirkwi_level": 1, + "label": { + "fr": "Très facile", + "en": "Very easy", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-1.svg" + }, + { + "id": 2, + "cirkwi_level": 2, + "label": { + "fr": "Facile", + "en": "Easy", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-2.svg" + }, + { + "id": 3, + "cirkwi_level": 3, + "label": { + "fr": "Intermédiaire", + "en": "Medium", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-3.svg" + }, + { + "id": 4, + "cirkwi_level": 4, + "label": { + "fr": "Difficile", + "en": "Hard", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-4.svg" + }, + { + "id": 5, + "cirkwi_level": 5, + "label": { + "fr": "Très difficile", + "en": "Very hard", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-5.svg" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json new file mode 100644 index 0000000000..d6370dbf47 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json @@ -0,0 +1,27 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 2, + "label": { + "fr": "PR", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/network-PR.svg" + }, + { + "id": 4, + "label": { + "fr": "VTT", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/network-VTT.svg" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json new file mode 100644 index 0000000000..00bd0e5eb7 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json @@ -0,0 +1,40 @@ +{ + "count": 3, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "name": { + "fr": "VTT", + "en": "Mountain Bike", + "es": null, + "it": null + }, + "order": 3, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/practice-mountainbike.svg" + }, + { + "id": 3, + "name": { + "fr": "Cheval", + "en": "Horse", + "es": null, + "it": null + }, + "order": 4, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/practice-horse.svg" + }, + { + "id": 4, + "name": { + "fr": "Pédestre", + "en": null, + "es": null, + "it": null + }, + "order": 1, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/practice-foot.svg" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json new file mode 100644 index 0000000000..08437b7f5c --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json @@ -0,0 +1,37 @@ +{ + "count": 3, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/route-loop.svg", + "route": { + "fr": "Boucle", + "en": null, + "es": null, + "it": null + } + }, + { + "id": 2, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/route-return.svg", + "route": { + "fr": "Aller-retour", + "en": null, + "es": null, + "it": null + } + }, + { + "id": 3, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/route-cross.svg", + "route": { + "fr": "Traversée", + "en": null, + "es": null, + "it": null + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json new file mode 100644 index 0000000000..69de2e43d0 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json @@ -0,0 +1,107 @@ +{ + "count": 10, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "label": { + "fr": "Faune", + "en": "Fauna", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-fauna.png" + }, + { + "id": 2, + "label": { + "fr": "Flore", + "en": "Flora", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-flora.png" + }, + { + "id": 4, + "label": { + "fr": "Point de vue", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-panorama.png" + }, + { + "id": 5, + "label": { + "fr": "Architecture", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-architecture.png" + }, + { + "id": 6, + "label": { + "fr": "Pastoralisme", + "en": "Pastoralism", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-pastoral.png" + }, + { + "id": 7, + "label": { + "fr": "Géologie", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-geology.png" + }, + { + "id": 8, + "label": { + "fr": "Lac et glacier", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-lake.png" + }, + { + "id": 9, + "label": { + "fr": "Sommet", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-peak.png" + }, + { + "id": 10, + "label": { + "fr": "Refuge", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-refugee.png" + }, + { + "id": 11, + "label": { + "fr": "Archéologie et histoire", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-history.png" + } + ] +} \ No newline at end of file From 170886d1348f6d68ab421d06547a6f627191ac32 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 17:13:51 +0200 Subject: [PATCH 013/116] Add test service parser api v2 --- .../tests/data/geotrek_parser_v2/service.json | 37 ++++++++++++++++ .../data/geotrek_parser_v2/service_type.json | 39 ++++++++++++++++ geotrek/trekking/tests/test_parsers.py | 44 ++++++++++++++++++- 3 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/service.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service.json new file mode 100644 index 0000000000..985d25bfd2 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service.json @@ -0,0 +1,37 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 4886, + "eid": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.4375926, + 42.8190752, + 1725.0 + ] + }, + "structure": "MC", + "type": 1, + "uuid": "1c67978b-9833-4f31-a384-c76a05eb2fbf" + }, + { + "id": 10393, + "eid": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.3985067, + 42.7560827, + 2100.0 + ] + }, + "structure": "CCCCC", + "type": 1, + "uuid": "d2acb966-d01c-4eb1-81f9-a9f490c8484a" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json new file mode 100644 index 0000000000..d013498f06 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json @@ -0,0 +1,39 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "name": { + "fr": "Eau potable", + "en": "Drinking water", + "es": null, + "it": null + }, + "practices": [ + 4, + 2, + 1, + 3 + ], + "pictogram": "https://demo-admin.geotrek.fr/media/upload/drinking_water.svg" + }, + { + "id": 2, + "name": { + "fr": "Descente pentue", + "en": "Steep descent", + "es": null, + "it": null + }, + "practices": [ + 4, + 2, + 1, + 3 + ], + "pictogram": "https://demo-admin.geotrek.fr/media/upload/steep_descent.svg" + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index a21343cb31..b189919008 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -10,8 +10,8 @@ from django.test.utils import override_settings from geotrek.common.models import Theme, FileType -from geotrek.trekking.models import POI, Trek, DifficultyLevel, Route -from geotrek.trekking.parsers import TrekParser, GeotrekPOIParser, GeotrekTrekParser +from geotrek.trekking.models import POI, Service, Trek, DifficultyLevel, Route +from geotrek.trekking.parsers import TrekParser, GeotrekPOIParser, GeotrekServiceParser, GeotrekTrekParser class TrekParserFilterDurationTests(TestCase): @@ -152,6 +152,14 @@ class TestGeotrekPOIParser(GeotrekPOIParser): } +class TestGeotrekServiceParser(GeotrekServiceParser): + url = "https://test.fr" + + field_options = { + 'type': {'create': True, }, + } + + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class TrekGeotrekParserTests(TestCase): @classmethod @@ -217,3 +225,35 @@ def mocked_json(): poi = POI.objects.all().first() self.assertEqual(poi.name, "Pic des Trois Seigneurs") self.assertEqual(str(poi.type), 'Sommet') + + +@skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') +class ServiceGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['service_type.json', 'service.json'] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekServiceParser', verbosity=0) + self.assertEqual(Service.objects.count(), 2) + service = Service.objects.all().first() + self.assertEqual(str(service.type), 'Eau potable') From 0f833d8968c0d6bc915f92d33cf4991df54e7c0b Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 18:46:39 +0200 Subject: [PATCH 014/116] Add tests tourism parser geotrek api v2 --- .../geotrek_parser_v2/informationdesk.json | 89 ++++ .../geotrek_parser_v2/touristiccontent.json | 198 +++++++++ .../touristiccontent_category.json | 385 ++++++++++++++++++ .../touristiccontent_themes.json | 107 +++++ .../geotrek_parser_v2/touristicevent.json | 160 ++++++++ .../touristicevent_type.json | 17 + geotrek/tourism/tests/test_parsers.py | 130 +++++- 7 files changed, 1085 insertions(+), 1 deletion(-) create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_type.json diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json new file mode 100644 index 0000000000..44a307b859 --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json @@ -0,0 +1,89 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "accessibility": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "description": { + "fr": "Informations diverses", + "en": "Various information", + "es": "Informaciones diversas", + "it": null + }, + "email": "", + "label_accessibility": null, + "latitude": 42.86365970075902, + "longitude": 1.2016725540161124, + "municipality": "Seix", + "name": { + "fr": "Office de Tourisme de Seix", + "en": "Seix Tourism Office", + "es": "Officio de Turismo de Seix", + "it": null + }, + "phone": "09 64 46 55 12", + "photo_url": "https://demo-admin.geotrek.fr/media/upload/office-seix.jpg.150x150_q85.jpg", + "postal_code": "09140", + "street": "33 Place Champ de Mars", + "type": { + "id": 1, + "label": { + "fr": "Maisons du parc", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/desktype-info.svg" + }, + "website": "https://www.google.fr/maps/place/Office+Tourisme/@42.8772264,1.207959,15z/data=!4m2!3m1!1s0x12a8caa4377c469d:0xf8f00e9b653bb18f" + }, + { + "id": 2, + "accessibility": { + "fr": "lkkll", + "en": "", + "es": "", + "it": "" + }, + "description": { + "fr": "Test description avec\r\n
    \r\n
  • des éléments HTML
    • \r\n
  • et des listes à
    • \r\n
  • puces
    • \r\n
", + "en": "", + "es": "", + "it": "" + }, + "email": null, + "label_accessibility": null, + "latitude": 43.58039085560773, + "longitude": 1.4282226562500002, + "municipality": "Toulouse", + "name": { + "fr": "Foo", + "en": null, + "es": null, + "it": null + }, + "phone": null, + "photo_url": "", + "postal_code": "31300", + "street": "52 rue Jacques Babinet", + "type": { + "id": 2, + "label": { + "fr": "Relais d'information", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/desktype-info.svg" + }, + "website": null + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json new file mode 100644 index 0000000000..879b6bb1cc --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json @@ -0,0 +1,198 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 10, + "accessibility": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "Wikimedia Common", + "license": null, + "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-01.jpg.400x0_q85.jpg", + "legend": "Ânes", + "title": "anes-01", + "url": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-01.jpg", + "uuid": "1f9887b3-dc75-46a3-a93e-e2c1a8ef7105" + }, + { + "backend": "", + "type": "image", + "author": "Wikimedia Commons", + "license": null, + "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-02.jpg.400x0_q85.jpg", + "legend": "Ânes", + "title": "anes-02", + "url": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-02.jpg", + "uuid": "36d83840-ed78-4d2e-9937-d259930f053e" + }, + { + "backend": "Attachment", + "type": "video", + "author": "", + "license": null, + "thumbnail": "", + "legend": "", + "title": "", + "url": "https://soundcloud.com/festivalnumerozero/radio-petit-zero-3-sortir-balade-jaune", + "uuid": "1918257f-19fb-4387-b117-d5ceb4a0f4ab" + } + ], + "approved": false, + "category": 3, + "description": { + "fr": "

Vacances à la ferme, Activités en pleine nature dans les Pyrénées.

\r\n

Nous vous accueillons en pleine montagne entre le massif des Trois Seigneurs et l’Étang de Lers en Ariège, Midi-Pyrénées. Passionnés par les ânes et la randonnée, nous élevons nos ânes depuis

\r\n

1998, qui sont éduqués et câlinés depuis tout petit.

\r\n

Partagez le plaisir de randonnée léger et libre avec nous et permettez aux enfants de découvrir

\r\n

une autre monde au milieu de la nature.

", + "en": "", + "es": "", + "it": null + }, + "description_teaser": { + "fr": "

Élevage et Éducation d'ânes, Séjours nature et montagne en Ariège, Accueil Cavaliers.

", + "en": "", + "es": "", + "it": null + }, + "departure_city": "09231", + "geometry": { + "type": "Point", + "coordinates": [ + 1.3877974, + 42.8597065 + ] + }, + "label_accessibility": null, + "practical_info": { + "fr": "", + "en": "", + "es": "", + "it": null + }, + "url": "https://demo-admin.geotrek.fr/api/v2/touristiccontent/10/", + "cities": [ + "09231" + ], + "create_datetime": "2015-12-30T14:12:50.965428+01:00", + "external_id": "", + "name": { + "fr": "Balad'âne", + "en": "", + "es": "", + "it": null + }, + "pdf": { + "fr": "https://demo-admin.geotrek.fr/api/fr/touristiccontents/10/baladane.pdf", + "en": "https://demo-admin.geotrek.fr/api/en/touristiccontents/10/baladane.pdf", + "es": "https://demo-admin.geotrek.fr/api/es/touristiccontents/10/baladane.pdf", + "it": "https://demo-admin.geotrek.fr/api/it/touristiccontents/10/baladane.pdf" + }, + "portal": [], + "published": true, + "source": [], + "structure": 1, + "themes": [ + 1, + 6 + ], + "update_datetime": "2015-12-30T14:17:31.483686+01:00", + "types": { + "301": [ + 24 + ], + "302": [ + 38 + ] + }, + "contact": "

Balad'âne - Salbis - 09320 Le Port - Tel : 33 (0)5 61 04 41 35

", + "email": "", + "website": "http://www.baladane.com/fr/", + "reservation_system": null, + "reservation_id": "", + "uuid": "921f9859-94b5-447a-ad20-946697323569" + }, + { + "id": 22, + "accessibility": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "attachments": [], + "approved": false, + "category": 5, + "description": { + "fr": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. ", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "Ce magnifique bar a requin est au beau milieu des pyrénées.", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09322", + "geometry": { + "type": "Point", + "coordinates": [ + 1.2808534, + 42.735508 + ] + }, + "label_accessibility": null, + "practical_info": { + "fr": "Attention à l'épilepsie", + "en": "", + "es": "", + "it": "" + }, + "url": "https://demo-admin.geotrek.fr/api/v2/touristiccontent/22/", + "cities": [ + "09322" + ], + "create_datetime": "2019-09-17T14:55:46.308511+02:00", + "external_id": null, + "name": { + "fr": "Bar à requin", + "en": "", + "es": "", + "it": "" + }, + "pdf": { + "fr": "https://demo-admin.geotrek.fr/api/fr/touristiccontents/22/bar-a-requin.pdf", + "en": "https://demo-admin.geotrek.fr/api/en/touristiccontents/22/bar-a-requin.pdf", + "es": "https://demo-admin.geotrek.fr/api/es/touristiccontents/22/bar-a-requin.pdf", + "it": "https://demo-admin.geotrek.fr/api/it/touristiccontents/22/bar-a-requin.pdf" + }, + "portal": [], + "published": true, + "source": [], + "structure": 3, + "themes": [ + 1 + ], + "update_datetime": "2019-09-17T15:04:54.043628+02:00", + "types": { + "501": [ + 43 + ], + "502": [] + }, + "contact": "Contacter Jean", + "email": null, + "website": "http://make-everything-ok.com/", + "reservation_system": null, + "reservation_id": "", + "uuid": "a9bf823a-a472-4eb2-b273-3db1355602d0" + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json new file mode 100644 index 0000000000..8946cc2a81 --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json @@ -0,0 +1,385 @@ +{ + "count": 7, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "label": { + "fr": "Hébergement", + "en": "Accommodation", + "es": null, + "it": null + }, + "order": 20, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-accommodation.svg", + "types": [ + { + "id": 101, + "label": { + "fr": "Type d'usage", + "en": "Type of use", + "es": null, + "it": null + }, + "values": [ + { + "id": 4, + "label": { + "fr": "Gite d'étape", + "en": "Stopover", + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 6, + "label": { + "fr": "Village de vacances", + "en": "Holiday village", + "es": null, + "it": null + }, + "pictogram": "/media/upload/Lieux_culturel_0rT7C2L.svg" + } + ] + }, + { + "id": 102, + "label": { + "fr": "Label", + "en": "Label", + "es": null, + "it": null + }, + "values": [] + } + ] + }, + { + "id": 2, + "label": { + "fr": "Pleine Nature", + "en": "Outdoor", + "es": null, + "it": null + }, + "order": 13, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-outdoor.svg", + "types": [ + { + "id": 201, + "label": { + "fr": "Type d'usage", + "en": "Type of use", + "es": null, + "it": null + }, + "values": [ + { + "id": 14, + "label": { + "fr": "Canyoning", + "en": "Canyoning", + "es": null, + "it": null + }, + "pictogram": "/media/upload/noun_canyon_3005142.png" + }, + { + "id": 17, + "label": { + "fr": "Course d'orientation", + "en": "Orienteering", + "es": null, + "it": null + }, + "pictogram": "/media/upload/noun_Running_65866.png" + } + ] + }, + { + "id": 202, + "label": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "values": [] + } + ] + }, + { + "id": 3, + "label": { + "fr": "Sorties", + "en": "Visits", + "es": null, + "it": null + }, + "order": 28, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-visits.svg", + "types": [ + { + "id": 301, + "label": { + "fr": "Type d'usage", + "en": "Type of use", + "es": null, + "it": null + }, + "values": [ + { + "id": 24, + "label": { + "fr": "Ane", + "en": "Donkey", + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 30, + "label": { + "fr": "Canoë-Kayak", + "en": "Canoe-Kayak", + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 29, + "label": { + "fr": "Canyoning", + "en": "Canyoning", + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 32, + "label": { + "fr": "Vol libre", + "en": "Free fly", + "es": null, + "it": null + }, + "pictogram": null + } + ] + }, + { + "id": 302, + "label": { + "fr": "Service", + "en": "Prestation", + "es": null, + "it": null + }, + "values": [ + { + "id": 38, + "label": { + "fr": "Accompagnée", + "en": "Guided", + "es": null, + "it": null + }, + "pictogram": null + } + ] + } + ] + }, + { + "id": 5, + "label": { + "fr": "Restaurants", + "en": "Restaurants", + "es": null, + "it": null + }, + "order": 30, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-restaurants.svg", + "types": [ + { + "id": 501, + "label": { + "fr": "Type d'usage", + "en": "Type of use", + "es": null, + "it": null + }, + "values": [ + { + "id": 43, + "label": { + "fr": "Restaurant", + "en": "Restaurant", + "es": null, + "it": null + }, + "pictogram": null + } + ] + }, + { + "id": 502, + "label": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "values": [] + } + ] + }, + { + "id": 7, + "label": { + "fr": "Séjours", + "en": "Destinations", + "es": null, + "it": null + }, + "order": 25, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-destination.svg", + "types": [ + { + "id": 701, + "label": { + "fr": "Thématique", + "en": "Theme", + "es": null, + "it": null + }, + "values": [ + { + "id": 49, + "label": { + "fr": "Bien être", + "en": "Wellness", + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 50, + "label": { + "fr": "Nature", + "en": "Nature", + "es": null, + "it": null + }, + "pictogram": null + } + ] + }, + { + "id": 702, + "label": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "values": [] + } + ] + }, + { + "id": 8, + "label": { + "fr": "Musée", + "en": "Museum", + "es": null, + "it": null + }, + "order": 40, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-museum.svg", + "types": [ + { + "id": 801, + "label": { + "fr": "Type d'usage", + "en": "Type of use", + "es": null, + "it": null + }, + "values": [ + { + "id": 58, + "label": { + "fr": "Lieu de visite", + "en": "Visit site", + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 57, + "label": { + "fr": "Musée", + "en": "Museum", + "es": null, + "it": null + }, + "pictogram": null + } + ] + }, + { + "id": 802, + "label": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "values": [] + } + ] + }, + { + "id": 9, + "label": { + "fr": "Course d'orientation", + "en": null, + "es": null, + "it": null + }, + "order": 12, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-sites_2.svg", + "types": [ + { + "id": 901, + "label": { + "fr": "Genre", + "en": null, + "es": null, + "it": null + }, + "values": [] + }, + { + "id": 902, + "label": { + "fr": "Label", + "en": null, + "es": null, + "it": null + }, + "values": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json new file mode 100644 index 0000000000..69de2e43d0 --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json @@ -0,0 +1,107 @@ +{ + "count": 10, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "label": { + "fr": "Faune", + "en": "Fauna", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-fauna.png" + }, + { + "id": 2, + "label": { + "fr": "Flore", + "en": "Flora", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-flora.png" + }, + { + "id": 4, + "label": { + "fr": "Point de vue", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-panorama.png" + }, + { + "id": 5, + "label": { + "fr": "Architecture", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-architecture.png" + }, + { + "id": 6, + "label": { + "fr": "Pastoralisme", + "en": "Pastoralism", + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-pastoral.png" + }, + { + "id": 7, + "label": { + "fr": "Géologie", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-geology.png" + }, + { + "id": 8, + "label": { + "fr": "Lac et glacier", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-lake.png" + }, + { + "id": 9, + "label": { + "fr": "Sommet", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-peak.png" + }, + { + "id": 10, + "label": { + "fr": "Refuge", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-refugee.png" + }, + { + "id": 11, + "label": { + "fr": "Archéologie et histoire", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-history.png" + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json new file mode 100644 index 0000000000..937001302a --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json @@ -0,0 +1,160 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 3, + "accessibility": { + "fr": "", + "en": "", + "es": "", + "it": null + }, + "approved": false, + "attachments": [], + "begin_date": "2025-07-31", + "booking": "", + "cities": [], + "contact": "

Autrefois le Couserans
Place Alphonse Sentein
09200 Saint-Girons
05 34 14 62 69
09 67 11 26 69

", + "create_datetime": "2015-03-25T14:09:44.346826+01:00", + "description": { + "fr": "

Ainsi défile dans les rues et sur les boulevards de la ville, la longue caravane constituée de plus d'une cinquantaine de vieux tracteurs, en parfait état de fonctionnement, mais également de vieilles machines agricoles , faucheuses, batteuses « qui dans la journée avaleront goulûment leurs gerbes de blés ». Tous ces matériels et engins, sans oublier les charrettes, chars et tombereaux, sont guidés par des hommes, des femmes et des enfants en costumes d'autrefois : boeufs, chevaux, ânes et mules tirent les attelages chargés de bois, de fourrage, de fumier...plus loin suivent d'autres animaux, moutons, dindons, oies et canards, chiens et derrière le cortège des lavandières, des couturières, des gardeuses d'oies, le livreur de lait, le facteur, le rémouleur, le marchand de pain, les porteurs d'eau, les marchandes des quatre saisons, la classe 1900, Monsieur le Curé et tant d'autres..

", + "en": "", + "es": "", + "it": null + }, + "description_teaser": { + "fr": "

Cette manifestation qui s’inscrit dorénavant dans le patrimoine Ariégeois, mobilise 800 participants et attire chaque premier week end d’Août  25 000 personnes venues de tout le Sud Ouest et de plus loin encore.

", + "en": "", + "es": "", + "it": null + }, + "duration": "3", + "email": "contact@autrefois-le-couserans.com", + "end_date": "2025-08-02", + "external_id": "", + "geometry": { + "type": "Point", + "coordinates": [ + 1.1496792, + 42.9637089 + ] + }, + "meeting_point": "Centre ville de Saint Girons", + "meeting_time": null, + "name": { + "fr": "Autrefois le Couserans", + "en": "", + "es": "", + "it": null + }, + "organizer": "", + "participant_number": "", + "pdf": { + "fr": "https://demo-admin.geotrek.fr/api/fr/touristicevents/3/autrefois-le-couserans.pdf", + "en": "https://demo-admin.geotrek.fr/api/en/touristicevents/3/autrefois-le-couserans.pdf", + "es": "https://demo-admin.geotrek.fr/api/es/touristicevents/3/autrefois-le-couserans.pdf", + "it": "https://demo-admin.geotrek.fr/api/it/touristicevents/3/autrefois-le-couserans.pdf" + }, + "portal": [], + "practical_info": { + "fr": "", + "en": "", + "es": "", + "it": null + }, + "published": true, + "source": [], + "speaker": "", + "structure": 1, + "target_audience": "", + "themes": [], + "type": 6, + "update_datetime": "2016-03-16T16:50:37.409679+01:00", + "url": "https://demo-admin.geotrek.fr/api/v2/touristicevent/3/", + "uuid": "7382c443-8206-4ef2-bf43-f471cdc5073d", + "website": "http://www.autrefois-le-couserans.com/" + }, + { + "id": 1, + "accessibility": { + "fr": "accessible aux handicapés moteurs", + "en": "", + "es": "", + "it": null + }, + "approved": false, + "attachments": [], + "begin_date": "2025-01-16", + "booking": "

Pour les réservations, voir l'encars contact

", + "cities": [ + "09100" + ], + "contact": "

Contactez moi, ou elle, ou lui !
06 12 12 12 12 12

", + "create_datetime": "2015-01-17T14:44:58.483513+01:00", + "description": { + "fr": "

Ceci est un test evenement tourristique

", + "en": "", + "es": "", + "it": null + }, + "description_teaser": { + "fr": "

Les evenements tourristiques sont supers

", + "en": "", + "es": "", + "it": null + }, + "duration": "premier semestre", + "email": "test@test.fr", + "end_date": "2025-04-16", + "external_id": "", + "geometry": { + "type": "Point", + "coordinates": [ + 1.1946237, + 42.7666746 + ] + }, + "meeting_point": "Au bout du monde", + "meeting_time": "15:45:00", + "name": { + "fr": "Festival du test", + "en": "", + "es": "", + "it": null + }, + "organizer": "Mr Test", + "participant_number": "12", + "pdf": { + "fr": "https://demo-admin.geotrek.fr/api/fr/touristicevents/1/festival-du-test.pdf", + "en": "https://demo-admin.geotrek.fr/api/en/touristicevents/1/festival-du-test.pdf", + "es": "https://demo-admin.geotrek.fr/api/es/touristicevents/1/festival-du-test.pdf", + "it": "https://demo-admin.geotrek.fr/api/it/touristicevents/1/festival-du-test.pdf" + }, + "portal": [], + "practical_info": { + "fr": "

Venez en bonne compagnie et avec le sourire !

", + "en": "", + "es": "", + "it": null + }, + "published": true, + "source": [], + "speaker": "Batman", + "structure": 1, + "target_audience": "Tout public", + "themes": [ + 5, + 7, + 4, + 10 + ], + "type": 6, + "update_datetime": "2016-03-16T16:50:09.774431+01:00", + "url": "https://demo-admin.geotrek.fr/api/v2/touristicevent/1/", + "uuid": "20fb113f-07f7-4deb-94b4-8445c85d5598", + "website": "http://test.fr/" + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_type.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_type.json new file mode 100644 index 0000000000..3911e20712 --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_type.json @@ -0,0 +1,17 @@ +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "id": 6, + "pictogram": null, + "type": { + "fr": "Spectacle", + "en": "Show", + "es": null, + "it": null + } + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 05905a248c..087d71dd48 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -19,7 +19,8 @@ from geotrek.tourism.parsers import (TouristicContentApidaeParser, TouristicEventApidaeParser, EspritParcParser, TouristicContentTourInSoftParserV3, TouristicContentTourInSoftParserV3withMedias, TouristicContentTourInSoftParser, TouristicEventTourInSoftParser, - InformationDeskApidaeParser) + InformationDeskApidaeParser, GeotrekTouristicContentParser, + GeotrekTouristicEventParser, GeotrekInformationDeskParser) class ApidaeConstantFieldContentParser(TouristicContentApidaeParser): @@ -647,3 +648,130 @@ def mocked_json(): information_desk_2 = InformationDesk.objects.get(eid=2) self.assertEqual(information_desk_2.website, None) + + +class TestGeotrekTouristicContentParser(GeotrekTouristicContentParser): + url = "https://test.fr" + + field_options = { + "category": {'create': True}, + 'themes': {'create': True}, + 'type1': {'create': True}, + 'type2': {'create': True} + } + + +class TestGeotrekTouristicEventParser(GeotrekTouristicEventParser): + url = "https://test.fr" + + field_options = { + 'type': {'create': True, }, + } + + +class TestGeotrekInformationDeskParser(GeotrekInformationDeskParser): + url = "https://test.fr" + + field_options = { + 'type': {'create': True, }, + } + + +class TouristicContentGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['touristiccontent_category.json', + 'touristiccontent_themes.json', + 'touristiccontent_category.json', + 'touristiccontent.json'] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekTouristicContentParser', verbosity=0) + self.assertEqual(TouristicContent.objects.count(), 2) + touristic_content = TouristicContent.objects.all().first() + self.assertEqual(str(touristic_content.category), 'Sorties') + self.assertEqual(str(touristic_content.name), "Balad'âne") + + +class TouristicEventGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['touristicevent_type.json', + 'touristicevent.json'] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekTouristicEventParser', verbosity=0) + self.assertEqual(TouristicEvent.objects.count(), 2) + touristic_event = TouristicEvent.objects.all().first() + self.assertEqual(str(touristic_event.type), 'Spectacle') + self.assertEqual(str(touristic_event.name), "Autrefois le Couserans") + + +class InformationDeskGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['informationdesk.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekInformationDeskParser', verbosity=0) + self.assertEqual(InformationDesk.objects.count(), 2) + information_desk = InformationDesk.objects.all().first() + self.assertEqual(str(information_desk.type), "Relais d'information") + self.assertEqual(str(information_desk.name), "Foo") From 4508d28c535192d1e5d8de6f485d7b3b25a8f780 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 29 Jul 2022 18:47:51 +0200 Subject: [PATCH 015/116] Fix url foo parsers --- .../geotrek_parser_v2/informationdesk.json | 6 +- .../geotrek_parser_v2/touristiccontent.json | 28 +++++----- .../touristiccontent_category.json | 14 ++--- .../touristiccontent_themes.json | 20 +++---- .../geotrek_parser_v2/touristicevent.json | 20 +++---- .../tests/data/geotrek_parser_v2/poi.json | 14 ++--- .../data/geotrek_parser_v2/poi_type.json | 32 +++++------ .../data/geotrek_parser_v2/service_type.json | 4 +- .../tests/data/geotrek_parser_v2/trek.json | 56 +++++++++---------- .../geotrek_parser_v2/trek_accessibility.json | 6 +- .../geotrek_parser_v2/trek_difficulty.json | 10 ++-- .../data/geotrek_parser_v2/trek_network.json | 4 +- .../data/geotrek_parser_v2/trek_practice.json | 6 +- .../data/geotrek_parser_v2/trek_route.json | 6 +- .../data/geotrek_parser_v2/trek_theme.json | 20 +++---- 15 files changed, 123 insertions(+), 123 deletions(-) diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json index 44a307b859..af7494757c 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json @@ -29,7 +29,7 @@ "it": null }, "phone": "09 64 46 55 12", - "photo_url": "https://demo-admin.geotrek.fr/media/upload/office-seix.jpg.150x150_q85.jpg", + "photo_url": "https://foo.fr/media/upload/office-seix.jpg.150x150_q85.jpg", "postal_code": "09140", "street": "33 Place Champ de Mars", "type": { @@ -40,7 +40,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/desktype-info.svg" + "pictogram": "https://foo.fr/media/upload/desktype-info.svg" }, "website": "https://www.google.fr/maps/place/Office+Tourisme/@42.8772264,1.207959,15z/data=!4m2!3m1!1s0x12a8caa4377c469d:0xf8f00e9b653bb18f" }, @@ -81,7 +81,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/desktype-info.svg" + "pictogram": "https://foo.fr/media/upload/desktype-info.svg" }, "website": null } diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json index 879b6bb1cc..e4feb8401b 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent.json @@ -17,10 +17,10 @@ "type": "image", "author": "Wikimedia Common", "license": null, - "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-01.jpg.400x0_q85.jpg", + "thumbnail": "https://foo.fr/media/paperclip/tourism_touristiccontent/10/anes-01.jpg.400x0_q85.jpg", "legend": "Ânes", "title": "anes-01", - "url": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-01.jpg", + "url": "https://foo.fr/media/paperclip/tourism_touristiccontent/10/anes-01.jpg", "uuid": "1f9887b3-dc75-46a3-a93e-e2c1a8ef7105" }, { @@ -28,10 +28,10 @@ "type": "image", "author": "Wikimedia Commons", "license": null, - "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-02.jpg.400x0_q85.jpg", + "thumbnail": "https://foo.fr/media/paperclip/tourism_touristiccontent/10/anes-02.jpg.400x0_q85.jpg", "legend": "Ânes", "title": "anes-02", - "url": "https://demo-admin.geotrek.fr/media/paperclip/tourism_touristiccontent/10/anes-02.jpg", + "url": "https://foo.fr/media/paperclip/tourism_touristiccontent/10/anes-02.jpg", "uuid": "36d83840-ed78-4d2e-9937-d259930f053e" }, { @@ -75,7 +75,7 @@ "es": "", "it": null }, - "url": "https://demo-admin.geotrek.fr/api/v2/touristiccontent/10/", + "url": "https://foo.fr/api/v2/touristiccontent/10/", "cities": [ "09231" ], @@ -88,10 +88,10 @@ "it": null }, "pdf": { - "fr": "https://demo-admin.geotrek.fr/api/fr/touristiccontents/10/baladane.pdf", - "en": "https://demo-admin.geotrek.fr/api/en/touristiccontents/10/baladane.pdf", - "es": "https://demo-admin.geotrek.fr/api/es/touristiccontents/10/baladane.pdf", - "it": "https://demo-admin.geotrek.fr/api/it/touristiccontents/10/baladane.pdf" + "fr": "https://foo.fr/api/fr/touristiccontents/10/baladane.pdf", + "en": "https://foo.fr/api/en/touristiccontents/10/baladane.pdf", + "es": "https://foo.fr/api/es/touristiccontents/10/baladane.pdf", + "it": "https://foo.fr/api/it/touristiccontents/10/baladane.pdf" }, "portal": [], "published": true, @@ -155,7 +155,7 @@ "es": "", "it": "" }, - "url": "https://demo-admin.geotrek.fr/api/v2/touristiccontent/22/", + "url": "https://foo.fr/api/v2/touristiccontent/22/", "cities": [ "09322" ], @@ -168,10 +168,10 @@ "it": "" }, "pdf": { - "fr": "https://demo-admin.geotrek.fr/api/fr/touristiccontents/22/bar-a-requin.pdf", - "en": "https://demo-admin.geotrek.fr/api/en/touristiccontents/22/bar-a-requin.pdf", - "es": "https://demo-admin.geotrek.fr/api/es/touristiccontents/22/bar-a-requin.pdf", - "it": "https://demo-admin.geotrek.fr/api/it/touristiccontents/22/bar-a-requin.pdf" + "fr": "https://foo.fr/api/fr/touristiccontents/22/bar-a-requin.pdf", + "en": "https://foo.fr/api/en/touristiccontents/22/bar-a-requin.pdf", + "es": "https://foo.fr/api/es/touristiccontents/22/bar-a-requin.pdf", + "it": "https://foo.fr/api/it/touristiccontents/22/bar-a-requin.pdf" }, "portal": [], "published": true, diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json index 8946cc2a81..67e843aa1a 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_category.json @@ -12,7 +12,7 @@ "it": null }, "order": 20, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-accommodation.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-accommodation.svg", "types": [ { "id": 101, @@ -66,7 +66,7 @@ "it": null }, "order": 13, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-outdoor.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-outdoor.svg", "types": [ { "id": 201, @@ -120,7 +120,7 @@ "it": null }, "order": 28, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-visits.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-visits.svg", "types": [ { "id": 301, @@ -205,7 +205,7 @@ "it": null }, "order": 30, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-restaurants.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-restaurants.svg", "types": [ { "id": 501, @@ -249,7 +249,7 @@ "it": null }, "order": 25, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-destination.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-destination.svg", "types": [ { "id": 701, @@ -303,7 +303,7 @@ "it": null }, "order": 40, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-museum.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-museum.svg", "types": [ { "id": 801, @@ -357,7 +357,7 @@ "it": null }, "order": 12, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/touristiccontent-sites_2.svg", + "pictogram": "https://foo.fr/media/upload/touristiccontent-sites_2.svg", "types": [ { "id": 901, diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json index 69de2e43d0..cc77967cf7 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_themes.json @@ -11,7 +11,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-fauna.png" + "pictogram": "https://foo.fr/media/upload/theme-fauna.png" }, { "id": 2, @@ -21,7 +21,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-flora.png" + "pictogram": "https://foo.fr/media/upload/theme-flora.png" }, { "id": 4, @@ -31,7 +31,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-panorama.png" + "pictogram": "https://foo.fr/media/upload/theme-panorama.png" }, { "id": 5, @@ -41,7 +41,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-architecture.png" + "pictogram": "https://foo.fr/media/upload/theme-architecture.png" }, { "id": 6, @@ -51,7 +51,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-pastoral.png" + "pictogram": "https://foo.fr/media/upload/theme-pastoral.png" }, { "id": 7, @@ -61,7 +61,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-geology.png" + "pictogram": "https://foo.fr/media/upload/theme-geology.png" }, { "id": 8, @@ -71,7 +71,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-lake.png" + "pictogram": "https://foo.fr/media/upload/theme-lake.png" }, { "id": 9, @@ -81,7 +81,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-peak.png" + "pictogram": "https://foo.fr/media/upload/theme-peak.png" }, { "id": 10, @@ -91,7 +91,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-refugee.png" + "pictogram": "https://foo.fr/media/upload/theme-refugee.png" }, { "id": 11, @@ -101,7 +101,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-history.png" + "pictogram": "https://foo.fr/media/upload/theme-history.png" } ] } \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json index 937001302a..d733bd5822 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent.json @@ -52,10 +52,10 @@ "organizer": "", "participant_number": "", "pdf": { - "fr": "https://demo-admin.geotrek.fr/api/fr/touristicevents/3/autrefois-le-couserans.pdf", - "en": "https://demo-admin.geotrek.fr/api/en/touristicevents/3/autrefois-le-couserans.pdf", - "es": "https://demo-admin.geotrek.fr/api/es/touristicevents/3/autrefois-le-couserans.pdf", - "it": "https://demo-admin.geotrek.fr/api/it/touristicevents/3/autrefois-le-couserans.pdf" + "fr": "https://foo.fr/api/fr/touristicevents/3/autrefois-le-couserans.pdf", + "en": "https://foo.fr/api/en/touristicevents/3/autrefois-le-couserans.pdf", + "es": "https://foo.fr/api/es/touristicevents/3/autrefois-le-couserans.pdf", + "it": "https://foo.fr/api/it/touristicevents/3/autrefois-le-couserans.pdf" }, "portal": [], "practical_info": { @@ -72,7 +72,7 @@ "themes": [], "type": 6, "update_datetime": "2016-03-16T16:50:37.409679+01:00", - "url": "https://demo-admin.geotrek.fr/api/v2/touristicevent/3/", + "url": "https://foo.fr/api/v2/touristicevent/3/", "uuid": "7382c443-8206-4ef2-bf43-f471cdc5073d", "website": "http://www.autrefois-le-couserans.com/" }, @@ -127,10 +127,10 @@ "organizer": "Mr Test", "participant_number": "12", "pdf": { - "fr": "https://demo-admin.geotrek.fr/api/fr/touristicevents/1/festival-du-test.pdf", - "en": "https://demo-admin.geotrek.fr/api/en/touristicevents/1/festival-du-test.pdf", - "es": "https://demo-admin.geotrek.fr/api/es/touristicevents/1/festival-du-test.pdf", - "it": "https://demo-admin.geotrek.fr/api/it/touristicevents/1/festival-du-test.pdf" + "fr": "https://foo.fr/api/fr/touristicevents/1/festival-du-test.pdf", + "en": "https://foo.fr/api/en/touristicevents/1/festival-du-test.pdf", + "es": "https://foo.fr/api/es/touristicevents/1/festival-du-test.pdf", + "it": "https://foo.fr/api/it/touristicevents/1/festival-du-test.pdf" }, "portal": [], "practical_info": { @@ -152,7 +152,7 @@ ], "type": 6, "update_datetime": "2016-03-16T16:50:09.774431+01:00", - "url": "https://demo-admin.geotrek.fr/api/v2/touristicevent/1/", + "url": "https://foo.fr/api/v2/touristicevent/1/", "uuid": "20fb113f-07f7-4deb-94b4-8445c85d5598", "website": "http://test.fr/" } diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json index b17abdcbfe..06bea73028 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json @@ -35,7 +35,7 @@ "thumbnail": "", "legend": "", "title": "", - "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_poi/2867/dep-a5-chartreuse-bat1.pdf", + "url": "https://foo.fr/media/paperclip/trekking_poi/2867/dep-a5-chartreuse-bat1.pdf", "uuid": "004c3484-d018-4706-996a-fba6e89abd96" }, { @@ -63,8 +63,8 @@ "es": null, "it": null }, - "type_pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-peak.png", - "url": "https://demo-admin.geotrek.fr/api/v2/poi/2867/", + "type_pictogram": "https://foo.fr/media/upload/poi-peak.png", + "url": "https://foo.fr/api/v2/poi/2867/", "uuid": "d4f534b5-74af-40a0-9c6c-825c385b4e93", "create_datetime": "2013-12-19T10:53:07.061717+01:00", "update_datetime": "2020-11-24T10:52:46.696698+01:00" @@ -98,10 +98,10 @@ "type": "image", "author": "ads", "license": null, - "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/trekking_poi/2869/sensitivearea.png.400x0_q85.png", + "thumbnail": "https://foo.fr/media/paperclip/trekking_poi/2869/sensitivearea.png.400x0_q85.png", "legend": "", "title": "sensitivearea", - "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_poi/2869/sensitivearea.png", + "url": "https://foo.fr/media/paperclip/trekking_poi/2869/sensitivearea.png", "uuid": "542afd02-487a-4df7-afaf-2082b24e8b6e" }, { @@ -129,8 +129,8 @@ "es": null, "it": null }, - "type_pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-lake.png", - "url": "https://demo-admin.geotrek.fr/api/v2/poi/2869/", + "type_pictogram": "https://foo.fr/media/upload/poi-lake.png", + "url": "https://foo.fr/api/v2/poi/2869/", "uuid": "3a48fd35-654a-442d-be37-cc4c43f620bb", "create_datetime": "2013-12-19T10:59:43.060709+01:00", "update_datetime": "2019-04-25T11:05:26.169812+02:00" diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json index 4ce0c17c30..f8e2004c3c 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json @@ -11,7 +11,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-archeology.png" + "pictogram": "https://foo.fr/media/upload/poi-archeology.png" }, { "id": 11, @@ -21,7 +21,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-architecture.png" + "pictogram": "https://foo.fr/media/upload/poi-architecture.png" }, { "id": 16, @@ -31,7 +31,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/POIE_point_attache.png" + "pictogram": "https://foo.fr/media/upload/POIE_point_attache.png" }, { "id": 15, @@ -41,7 +41,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-pass.png" + "pictogram": "https://foo.fr/media/upload/poi-pass.png" }, { "id": 3, @@ -51,7 +51,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-fauna.png" + "pictogram": "https://foo.fr/media/upload/poi-fauna.png" }, { "id": 2, @@ -61,7 +61,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-flora.png" + "pictogram": "https://foo.fr/media/upload/poi-flora.png" }, { "id": 8, @@ -71,7 +71,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-geology.png" + "pictogram": "https://foo.fr/media/upload/poi-geology.png" }, { "id": 12, @@ -81,7 +81,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-glacier.png" + "pictogram": "https://foo.fr/media/upload/poi-glacier.png" }, { "id": 10, @@ -91,7 +91,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-history.png" + "pictogram": "https://foo.fr/media/upload/poi-history.png" }, { "id": 6, @@ -101,7 +101,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-lake.png" + "pictogram": "https://foo.fr/media/upload/poi-lake.png" }, { "id": 14, @@ -111,7 +111,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-pastoral.png" + "pictogram": "https://foo.fr/media/upload/poi-pastoral.png" }, { "id": 1, @@ -121,7 +121,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-patrimony.png" + "pictogram": "https://foo.fr/media/upload/poi-patrimony.png" }, { "id": 4, @@ -131,7 +131,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-panorama.png" + "pictogram": "https://foo.fr/media/upload/poi-panorama.png" }, { "id": 5, @@ -141,7 +141,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-refuge.png" + "pictogram": "https://foo.fr/media/upload/poi-refuge.png" }, { "id": 13, @@ -151,7 +151,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-knowhow.png" + "pictogram": "https://foo.fr/media/upload/poi-knowhow.png" }, { "id": 7, @@ -161,7 +161,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/poi-peak.png" + "pictogram": "https://foo.fr/media/upload/poi-peak.png" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json index d013498f06..6ab2f72b26 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json @@ -17,7 +17,7 @@ 1, 3 ], - "pictogram": "https://demo-admin.geotrek.fr/media/upload/drinking_water.svg" + "pictogram": "https://foo.fr/media/upload/drinking_water.svg" }, { "id": 2, @@ -33,7 +33,7 @@ 1, 3 ], - "pictogram": "https://demo-admin.geotrek.fr/media/upload/steep_descent.svg" + "pictogram": "https://foo.fr/media/upload/steep_descent.svg" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json index f0668be3ab..6e49e92eae 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -83,10 +83,10 @@ "type": "image", "author": "Grégory Tonon", "license": null, - "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg.400x0_q85.jpg", + "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg.400x0_q85.jpg", "legend": "Vue sur la chaîne des Pyrénées, depuis le pics des 3 Seigneurs", "title": "1024px-Les_Pyrénées_depuis_le_pic_des_3_Seigneurs", - "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg", + "url": "https://foo.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg", "uuid": "8c62ae5f-9533-4de6-a863-3e33cd42d16c" }, { @@ -108,7 +108,7 @@ "thumbnail": "", "legend": "", "title": "file_example_MP3_700KB", - "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/file_example_mp3_700kb.mp3", + "url": "https://foo.fr/media/paperclip/trekking_trek/2/file_example_mp3_700kb.mp3", "uuid": "529e754b-e5a8-4fe7-944d-161b4900a62e" } ], @@ -134,7 +134,7 @@ ], "descent": -777, "description": { - "fr": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/2/file_example_mp3_700kb.mp3", + "fr": "https://foo.fr/media/paperclip/trekking_trek/2/file_example_mp3_700kb.mp3", "en": "", "es": "", "it": "" @@ -153,8 +153,8 @@ "it": "" }, "duration": 24.0, - "elevation_area_url": "https://demo-admin.geotrek.fr/api/v2/trek/2/dem/", - "elevation_svg_url": "https://demo-admin.geotrek.fr/api/v2/trek/2/profile/?language=fr&format=svg", + "elevation_area_url": "https://foo.fr/api/v2/trek/2/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/2/profile/?language=fr&format=svg", "external_id": null, "gear": { "fr": "Il faut de la corde", @@ -2072,13 +2072,13 @@ ] ] }, - "gpx": "https://demo-admin.geotrek.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.gpx", + "gpx": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.gpx", "information_desks": [ 2, 1, 3 ], - "kml": "https://demo-admin.geotrek.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.kml", + "kml": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.kml", "labels": [ 1, 2, @@ -2104,10 +2104,10 @@ 42.8063269 ], "pdf": { - "fr": "https://demo-admin.geotrek.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", - "en": "https://demo-admin.geotrek.fr/api/en/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", - "es": "https://demo-admin.geotrek.fr/api/es/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", - "it": "https://demo-admin.geotrek.fr/api/it/treks/2/boucle-du-pic-des-trois-seigneurs.pdf" + "fr": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "en": "https://foo.fr/api/en/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "es": "https://foo.fr/api/es/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "it": "https://foo.fr/api/it/treks/2/boucle-du-pic-des-trois-seigneurs.pdf" }, "points_reference": { "type": "MultiPoint", @@ -2171,7 +2171,7 @@ 4 ], "update_datetime": "2022-05-16T12:10:59.927409Z", - "url": "https://demo-admin.geotrek.fr/api/v2/trek/2/", + "url": "https://foo.fr/api/v2/trek/2/", "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1", "web_links": [ { @@ -2190,7 +2190,7 @@ "it": null }, "id": 4, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/weblink-book.png" + "pictogram": "https://foo.fr/media/upload/weblink-book.png" } }, { @@ -2209,7 +2209,7 @@ "it": null }, "id": 4, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/weblink-book.png" + "pictogram": "https://foo.fr/media/upload/weblink-book.png" } }, { @@ -2228,7 +2228,7 @@ "it": null }, "id": 4, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/weblink-book.png" + "pictogram": "https://foo.fr/media/upload/weblink-book.png" } } ] @@ -2291,7 +2291,7 @@ "es": "", "it": "" }, - "altimetric_profile": "https://demo-admin.geotrek.fr/api/v2/trek/10443/profile/", + "altimetric_profile": "https://foo.fr/api/v2/trek/10443/profile/", "ambiance": { "fr": "Test ambiance", "en": "", @@ -2311,10 +2311,10 @@ "type": "image", "author": "Samrong01 - CC BY-SA 4.0", "license": null, - "thumbnail": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/10443/1083px-arrien-en-bethmale_general_view.JPG.400x0_q85.jpg", + "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/10443/1083px-arrien-en-bethmale_general_view.JPG.400x0_q85.jpg", "legend": "Arrien-en-Bethmale, vue du village", "title": "", - "url": "https://demo-admin.geotrek.fr/media/paperclip/trekking_trek/10443/1083px-arrien-en-bethmale_general_view.JPG", + "url": "https://foo.fr/media/paperclip/trekking_trek/10443/1083px-arrien-en-bethmale_general_view.JPG", "uuid": "8605f89a-48fa-40ba-a9b0-9a48e5b0f310" } ], @@ -2357,8 +2357,8 @@ "it": "" }, "duration": 2.0, - "elevation_area_url": "https://demo-admin.geotrek.fr/api/v2/trek/10443/dem/", - "elevation_svg_url": "https://demo-admin.geotrek.fr/api/v2/trek/10443/profile/?language=fr&format=svg", + "elevation_area_url": "https://foo.fr/api/v2/trek/10443/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/10443/profile/?language=fr&format=svg", "external_id": null, "gear": { "fr": "", @@ -3016,9 +3016,9 @@ ] ] }, - "gpx": "https://demo-admin.geotrek.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.gpx", + "gpx": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.gpx", "information_desks": [], - "kml": "https://demo-admin.geotrek.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.kml", + "kml": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.kml", "labels": [], "length_2d": 3063.4, "length_3d": 3245.4, @@ -3039,10 +3039,10 @@ ], "parking_location": null, "pdf": { - "fr": "https://demo-admin.geotrek.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.pdf", - "en": "https://demo-admin.geotrek.fr/api/en/treks/10443/de-bethmale-au-col-de-la-core.pdf", - "es": "https://demo-admin.geotrek.fr/api/es/treks/10443/de-bethmale-au-col-de-la-core.pdf", - "it": "https://demo-admin.geotrek.fr/api/it/treks/10443/de-bethmale-au-col-de-la-core.pdf" + "fr": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "en": "https://foo.fr/api/en/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "es": "https://foo.fr/api/es/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "it": "https://foo.fr/api/it/treks/10443/de-bethmale-au-col-de-la-core.pdf" }, "points_reference": null, "portal": [], @@ -3077,7 +3077,7 @@ "structure": 3, "themes": [], "update_datetime": "2022-07-26T08:18:06.997582Z", - "url": "https://demo-admin.geotrek.fr/api/v2/trek/10443/", + "url": "https://foo.fr/api/v2/trek/10443/", "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12", "web_links": [] } diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json index 17fe26726c..5d26de4998 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json @@ -11,7 +11,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/accessibility-wheelchair.png" + "pictogram": "https://foo.fr/media/upload/accessibility-wheelchair.png" }, { "id": 2, @@ -21,7 +21,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/accessibility-troller.png" + "pictogram": "https://foo.fr/media/upload/accessibility-troller.png" }, { "id": 3, @@ -31,7 +31,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/accessibility-joelette.png" + "pictogram": "https://foo.fr/media/upload/accessibility-joelette.png" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json index b8f3236290..d361cd9c5a 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json @@ -12,7 +12,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-1.svg" + "pictogram": "https://foo.fr/media/upload/difficulty-1.svg" }, { "id": 2, @@ -23,7 +23,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-2.svg" + "pictogram": "https://foo.fr/media/upload/difficulty-2.svg" }, { "id": 3, @@ -34,7 +34,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-3.svg" + "pictogram": "https://foo.fr/media/upload/difficulty-3.svg" }, { "id": 4, @@ -45,7 +45,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-4.svg" + "pictogram": "https://foo.fr/media/upload/difficulty-4.svg" }, { "id": 5, @@ -56,7 +56,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/difficulty-5.svg" + "pictogram": "https://foo.fr/media/upload/difficulty-5.svg" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json index d6370dbf47..7548a9b966 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json @@ -11,7 +11,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/network-PR.svg" + "pictogram": "https://foo.fr/media/upload/network-PR.svg" }, { "id": 4, @@ -21,7 +21,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/network-VTT.svg" + "pictogram": "https://foo.fr/media/upload/network-VTT.svg" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json index 00bd0e5eb7..9f0d40c272 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json @@ -12,7 +12,7 @@ "it": null }, "order": 3, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/practice-mountainbike.svg" + "pictogram": "https://foo.fr/media/upload/practice-mountainbike.svg" }, { "id": 3, @@ -23,7 +23,7 @@ "it": null }, "order": 4, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/practice-horse.svg" + "pictogram": "https://foo.fr/media/upload/practice-horse.svg" }, { "id": 4, @@ -34,7 +34,7 @@ "it": null }, "order": 1, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/practice-foot.svg" + "pictogram": "https://foo.fr/media/upload/practice-foot.svg" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json index 08437b7f5c..0deba02309 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json @@ -5,7 +5,7 @@ "results": [ { "id": 1, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/route-loop.svg", + "pictogram": "https://foo.fr/media/upload/route-loop.svg", "route": { "fr": "Boucle", "en": null, @@ -15,7 +15,7 @@ }, { "id": 2, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/route-return.svg", + "pictogram": "https://foo.fr/media/upload/route-return.svg", "route": { "fr": "Aller-retour", "en": null, @@ -25,7 +25,7 @@ }, { "id": 3, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/route-cross.svg", + "pictogram": "https://foo.fr/media/upload/route-cross.svg", "route": { "fr": "Traversée", "en": null, diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json index 69de2e43d0..cc77967cf7 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json @@ -11,7 +11,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-fauna.png" + "pictogram": "https://foo.fr/media/upload/theme-fauna.png" }, { "id": 2, @@ -21,7 +21,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-flora.png" + "pictogram": "https://foo.fr/media/upload/theme-flora.png" }, { "id": 4, @@ -31,7 +31,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-panorama.png" + "pictogram": "https://foo.fr/media/upload/theme-panorama.png" }, { "id": 5, @@ -41,7 +41,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-architecture.png" + "pictogram": "https://foo.fr/media/upload/theme-architecture.png" }, { "id": 6, @@ -51,7 +51,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-pastoral.png" + "pictogram": "https://foo.fr/media/upload/theme-pastoral.png" }, { "id": 7, @@ -61,7 +61,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-geology.png" + "pictogram": "https://foo.fr/media/upload/theme-geology.png" }, { "id": 8, @@ -71,7 +71,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-lake.png" + "pictogram": "https://foo.fr/media/upload/theme-lake.png" }, { "id": 9, @@ -81,7 +81,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-peak.png" + "pictogram": "https://foo.fr/media/upload/theme-peak.png" }, { "id": 10, @@ -91,7 +91,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-refugee.png" + "pictogram": "https://foo.fr/media/upload/theme-refugee.png" }, { "id": 11, @@ -101,7 +101,7 @@ "es": null, "it": null }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/theme-history.png" + "pictogram": "https://foo.fr/media/upload/theme-history.png" } ] } \ No newline at end of file From 9589ad047ae0eb3260d38089c9037b2e8a386f53 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 1 Aug 2022 12:02:59 +0200 Subject: [PATCH 016/116] Fix parser common/infrastructure/signage, add tests signage/ infrastructure --- geotrek/infrastructure/parsers.py | 7 +- .../geotrek_parser_v2/infrastructure.json | 75 +++++++++ .../infrastructure_condition.json | 32 ++++ .../infrastructure_type.json | 28 ++++ geotrek/infrastructure/tests/test_parsers.py | 53 +++++++ geotrek/signage/parsers.py | 7 +- .../tests/data/geotrek_parser_v2/signage.json | 149 ++++++++++++++++++ .../geotrek_parser_v2/signage_condition.json | 32 ++++ .../geotrek_parser_v2/signage_sealing.json | 17 ++ .../data/geotrek_parser_v2/signage_type.json | 37 +++++ geotrek/signage/tests/test_parsers.py | 55 +++++++ 11 files changed, 482 insertions(+), 10 deletions(-) create mode 100644 geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure.json create mode 100644 geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_condition.json create mode 100644 geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_type.json create mode 100644 geotrek/infrastructure/tests/test_parsers.py create mode 100644 geotrek/signage/tests/data/geotrek_parser_v2/signage.json create mode 100644 geotrek/signage/tests/data/geotrek_parser_v2/signage_condition.json create mode 100644 geotrek/signage/tests/data/geotrek_parser_v2/signage_sealing.json create mode 100644 geotrek/signage/tests/data/geotrek_parser_v2/signage_type.json create mode 100644 geotrek/signage/tests/test_parsers.py diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py index 9fcc5b3316..7cb727a80c 100644 --- a/geotrek/infrastructure/parsers.py +++ b/geotrek/infrastructure/parsers.py @@ -10,7 +10,8 @@ class GeotrekInfrastructureParser(GeotrekParser): "deleted": False } replace_fields = { - "eid": "uuid" + "eid": "uuid", + "geom": "geometry" } url_categories = { 'condition': '/api/v2/infrastructure_condition/', @@ -25,10 +26,6 @@ class GeotrekInfrastructureParser(GeotrekParser): 'type': 'label' } - field_options = { - "type": {"create": True} - } - def next_row(self): self.next_url = f"{self.url}/api/v2/infrastructure" return super().next_row() diff --git a/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure.json b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure.json new file mode 100644 index 0000000000..da8bf477e7 --- /dev/null +++ b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure.json @@ -0,0 +1,75 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 2898, + "accessibility": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "gutard", + "license": null, + "thumbnail": "https://foo.fr/media/paperclip/infrastructure_baseinfrastructure/2898/belgique.png.400x0_q85.jpg", + "legend": "", + "title": "belgique", + "url": "https://foo.fr/media/paperclip/infrastructure_baseinfrastructure/2898/belgique.png", + "uuid": "20888c5a-0f33-4e22-8999-5fef740a5439" + } + ], + "condition": null, + "description": "", + "eid": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.35208, + 42.781068, + 840.0 + ] + }, + "name": "Table pic-nique", + "implantation_year": null, + "maintenance_difficulty": null, + "structure": "MC", + "type": 81, + "usage_difficulty": null, + "uuid": "93747e51-757e-4b39-9c3b-41bb228a7455" + }, + { + "id": 8385, + "accessibility": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "attachments": [], + "condition": null, + "description": "Une belle description", + "eid": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.1090391, + 42.7838115, + 1550.0 + ] + }, + "name": "Cabane d'Aula", + "implantation_year": null, + "maintenance_difficulty": null, + "structure": "DEMO", + "type": 45, + "usage_difficulty": null, + "uuid": "f7d10e86-5bf0-47a5-934e-63dcca8bf9c5" + } + ] +} \ No newline at end of file diff --git a/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_condition.json b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_condition.json new file mode 100644 index 0000000000..6d14c903b4 --- /dev/null +++ b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_condition.json @@ -0,0 +1,32 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 6, + "label": "Neuf", + "structure": null + }, + { + "id": 7, + "label": "Bon état", + "structure": null + }, + { + "id": 8, + "label": "Dégradé", + "structure": null + }, + { + "id": 9, + "label": "En ruines", + "structure": null + }, + { + "id": 10, + "label": "Tagué", + "structure": null + } + ] +} \ No newline at end of file diff --git a/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_type.json b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_type.json new file mode 100644 index 0000000000..2854eda972 --- /dev/null +++ b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_type.json @@ -0,0 +1,28 @@ +{ + "count": 3, + "next": null, + "previous": null, + "results": [ + { + "id": 45, + "label": "Aire de stationnement", + "pictogram": null, + "structure": 3, + "type": "Ouvrage" + }, + { + "id": 46, + "label": "Aire de stationnement Handicapés", + "pictogram": null, + "structure": null, + "type": "Ouvrage" + }, + { + "id": 81, + "label": "Table", + "pictogram": null, + "structure": null, + "type": "Équipement" + } + ] +} \ No newline at end of file diff --git a/geotrek/infrastructure/tests/test_parsers.py b/geotrek/infrastructure/tests/test_parsers.py new file mode 100644 index 0000000000..9a6e2f1ead --- /dev/null +++ b/geotrek/infrastructure/tests/test_parsers.py @@ -0,0 +1,53 @@ +from unittest import mock +import json +import os + +from django.core.management import call_command +from django.test import TestCase +from django.test.utils import override_settings + +from geotrek.common.models import FileType +from geotrek.infrastructure.models import Infrastructure +from geotrek.infrastructure.parsers import GeotrekInfrastructureParser + + +class TestGeotrekInfrastructureParser(GeotrekInfrastructureParser): + url = "https://test.fr" + + field_options = { + 'condition': {'create': True, }, + 'type': {'create': True}, + } + + +class InfrastructureGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['infrastructure_condition.json', 'infrastructure_type.json', 'infrastructure.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.infrastructure.tests.test_parsers.TestGeotrekInfrastructureParser', verbosity=0) + self.assertEqual(Infrastructure.objects.count(), 2) + infrastructure = Infrastructure.objects.all().first() + self.assertEqual(str(infrastructure.name), 'Table pic-nique') + self.assertEqual(str(infrastructure.type), 'Table') + self.assertEqual(str(infrastructure.geom.ewkt), 'SRID=2154;POINT (565008.6693905985 6188246.533542466)') diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py index f7befc4e71..86ec5a3cc9 100644 --- a/geotrek/signage/parsers.py +++ b/geotrek/signage/parsers.py @@ -10,7 +10,8 @@ class GeotrekSignageParser(GeotrekParser): "deleted": False } replace_fields = { - "eid": "uuid" + "eid": "uuid", + "geom": "geometry" } url_categories = { 'sealing': '/api/v2/signage_sealing/', @@ -28,10 +29,6 @@ class GeotrekSignageParser(GeotrekParser): 'type': 'label' } - field_options = { - "type": {"create": True} - } - def next_row(self): self.next_url = f"{self.url}/api/v2/signage" return super().next_row() diff --git a/geotrek/signage/tests/data/geotrek_parser_v2/signage.json b/geotrek/signage/tests/data/geotrek_parser_v2/signage.json new file mode 100644 index 0000000000..b01ba09978 --- /dev/null +++ b/geotrek/signage/tests/data/geotrek_parser_v2/signage.json @@ -0,0 +1,149 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 8416, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "gard", + "license": null, + "thumbnail": "https://foo.fr/media/paperclip/signage_signage/8416/2018-12-13-151001.jpg.400x0_q85.jpg", + "legend": "", + "title": "2018-12-13 15.10.01", + "url": "https://foo.fr/media/paperclip/signage_signage/8416/2018-12-13-151001.jpg", + "uuid": "ca59481d-9f99-40fa-acad-1e10ba323697" + } + ], + "blades": [ + { + "id": 2, + "number": "2", + "color": 2, + "direction": 2, + "lines": [ + { + "id": 3, + "text": "Joli col", + "pictogram": null, + "distance": "5.0", + "time": null + }, + { + "id": 4, + "text": "Joli sommet", + "pictogram": null, + "distance": "2.3", + "time": null + }, + { + "id": 5, + "text": "Joli lac", + "pictogram": null, + "distance": "0.6", + "time": null + } + ] + }, + { + "id": 5, + "number": "3", + "color": 2, + "direction": 2, + "lines": [ + { + "id": 10, + "text": "Toto la-bas", + "pictogram": null, + "distance": "14.0", + "time": null + } + ] + }, + { + "id": 11, + "number": "12", + "color": 1, + "direction": 1, + "lines": [ + { + "id": 20, + "text": "test", + "pictogram": null, + "distance": "2.0", + "time": null + } + ] + }, + { + "id": 1, + "number": "1", + "color": 1, + "direction": 2, + "lines": [ + { + "id": 2, + "text": "Les châlet", + "pictogram": null, + "distance": null, + "time": "01:45:00" + }, + { + "id": 1, + "text": "Trifouilli", + "pictogram": "GR6", + "distance": "2.7", + "time": "00:15:00" + } + ] + } + ], + "code": "30123/01", + "condition": 8, + "description": "des", + "eid": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.4487099, + 42.7892878, + 1160.0 + ] + }, + "implantation_year": 2018, + "name": "test gard", + "printed_elevation": null, + "sealing": 1, + "structure": "DEMO", + "type": 4, + "uuid": "6de1205c-8066-4e8b-8f84-f8736db61e95" + }, + { + "id": 8455, + "attachments": [], + "blades": [], + "code": "", + "condition": null, + "description": "", + "eid": null, + "geometry": { + "type": "Point", + "coordinates": [ + 1.4512499, + 42.7880983, + 1145.0 + ] + }, + "implantation_year": 2018, + "name": "Test picto defaut", + "printed_elevation": null, + "sealing": null, + "structure": "MC", + "type": 3, + "uuid": "133e9666-5bb5-4690-8034-6171bf4af115" + } + ] +} \ No newline at end of file diff --git a/geotrek/signage/tests/data/geotrek_parser_v2/signage_condition.json b/geotrek/signage/tests/data/geotrek_parser_v2/signage_condition.json new file mode 100644 index 0000000000..6d14c903b4 --- /dev/null +++ b/geotrek/signage/tests/data/geotrek_parser_v2/signage_condition.json @@ -0,0 +1,32 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 6, + "label": "Neuf", + "structure": null + }, + { + "id": 7, + "label": "Bon état", + "structure": null + }, + { + "id": 8, + "label": "Dégradé", + "structure": null + }, + { + "id": 9, + "label": "En ruines", + "structure": null + }, + { + "id": 10, + "label": "Tagué", + "structure": null + } + ] +} \ No newline at end of file diff --git a/geotrek/signage/tests/data/geotrek_parser_v2/signage_sealing.json b/geotrek/signage/tests/data/geotrek_parser_v2/signage_sealing.json new file mode 100644 index 0000000000..32d464871d --- /dev/null +++ b/geotrek/signage/tests/data/geotrek_parser_v2/signage_sealing.json @@ -0,0 +1,17 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "label": "Socle béton", + "structure": null + }, + { + "id": 2, + "label": "Applique", + "structure": null + } + ] +} \ No newline at end of file diff --git a/geotrek/signage/tests/data/geotrek_parser_v2/signage_type.json b/geotrek/signage/tests/data/geotrek_parser_v2/signage_type.json new file mode 100644 index 0000000000..f8c799d4aa --- /dev/null +++ b/geotrek/signage/tests/data/geotrek_parser_v2/signage_type.json @@ -0,0 +1,37 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 3, + "label": "Information", + "pictogram": null, + "structure": null + }, + { + "id": 4, + "label": "Limite Cœur", + "pictogram": null, + "structure": null + }, + { + "id": 5, + "label": "Porte d'entrée du Parc", + "pictogram": "https://foo.fr/media/upload/Lieux_culturel.svg", + "structure": null + }, + { + "id": 6, + "label": "Réglementaire", + "pictogram": null, + "structure": null + }, + { + "id": 9, + "label": "Temporaire", + "pictogram": null, + "structure": null + } + ] +} \ No newline at end of file diff --git a/geotrek/signage/tests/test_parsers.py b/geotrek/signage/tests/test_parsers.py new file mode 100644 index 0000000000..d360111521 --- /dev/null +++ b/geotrek/signage/tests/test_parsers.py @@ -0,0 +1,55 @@ +from unittest import mock +import json +import os + +from django.core.management import call_command +from django.test import TestCase +from django.test.utils import override_settings + +from geotrek.common.models import FileType +from geotrek.signage.models import Signage +from geotrek.signage.parsers import GeotrekSignageParser + + +class TestGeotrekSignageParser(GeotrekSignageParser): + url = "https://test.fr" + + field_options = { + 'sealing': {'create': True, }, + 'condition': {'create': True, }, + 'type': {'create': True}, + } + + +class SignageGeotrekParserTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.filetype = FileType.objects.create(type="Photographie") + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['signage_sealing.json', 'signage_condition.json', 'signage_type.json', 'signage.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.signage.tests.test_parsers.TestGeotrekSignageParser', verbosity=0) + self.assertEqual(Signage.objects.count(), 2) + signage = Signage.objects.all().first() + self.assertEqual(str(signage.name), 'test gard') + self.assertEqual(str(signage.type), 'Limite Cœur') + self.assertEqual(str(signage.sealing), 'Socle béton') + self.assertEqual(str(signage.geom.ewkt), 'SRID=2154;POINT (572941.1308660918 6189000.155980503)') From 2bb5988396cca5083a49f0fbd83bf61d4ce82484 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 1 Aug 2022 15:18:52 +0200 Subject: [PATCH 017/116] Add geom required in parsers --- geotrek/infrastructure/tests/test_parsers.py | 1 + geotrek/signage/tests/test_parsers.py | 1 + geotrek/tourism/parsers.py | 10 +++++----- geotrek/tourism/tests/test_parsers.py | 7 +++++-- geotrek/trekking/tests/test_parsers.py | 3 +++ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/geotrek/infrastructure/tests/test_parsers.py b/geotrek/infrastructure/tests/test_parsers.py index 9a6e2f1ead..84e209a6cf 100644 --- a/geotrek/infrastructure/tests/test_parsers.py +++ b/geotrek/infrastructure/tests/test_parsers.py @@ -17,6 +17,7 @@ class TestGeotrekInfrastructureParser(GeotrekInfrastructureParser): field_options = { 'condition': {'create': True, }, 'type': {'create': True}, + 'geom': {'required': True}, } diff --git a/geotrek/signage/tests/test_parsers.py b/geotrek/signage/tests/test_parsers.py index d360111521..e1f6a4bf75 100644 --- a/geotrek/signage/tests/test_parsers.py +++ b/geotrek/signage/tests/test_parsers.py @@ -18,6 +18,7 @@ class TestGeotrekSignageParser(GeotrekSignageParser): 'sealing': {'create': True, }, 'condition': {'create': True, }, 'type': {'create': True}, + 'geom': {'required': True} } diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 4562f9cb70..26f219abc1 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -926,6 +926,9 @@ class GeotrekTouristicContentParser(GeotrekParser): } field_options = { + 'type1': {'fk': 'category'}, + 'type2': {'fk': 'category'}, + 'geom': {'required': True}, } def __init__(self, *args, **kwargs): @@ -984,10 +987,6 @@ class GeotrekTouristicEventParser(GeotrekParser): 'type': 'type', } - field_options = { - "type": {"create": True} - } - def next_row(self): self.next_url = f"{self.url}/api/v2/touristicevent" return super().next_row() @@ -1009,7 +1008,8 @@ class GeotrekInformationDeskParser(GeotrekParser): } field_options = { - "type": {"create": True} + "type": {"create": True}, + 'geom': {'required': True}, } def next_row(self): diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 087d71dd48..4a15298a4f 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -656,8 +656,9 @@ class TestGeotrekTouristicContentParser(GeotrekTouristicContentParser): field_options = { "category": {'create': True}, 'themes': {'create': True}, - 'type1': {'create': True}, - 'type2': {'create': True} + 'type1': {'create': True, 'fk': 'category'}, + 'type2': {'create': True, 'fk': 'category'}, + 'geom': {'required': True}, } @@ -666,6 +667,7 @@ class TestGeotrekTouristicEventParser(GeotrekTouristicEventParser): field_options = { 'type': {'create': True, }, + 'geom': {'required': True}, } @@ -674,6 +676,7 @@ class TestGeotrekInformationDeskParser(GeotrekInformationDeskParser): field_options = { 'type': {'create': True, }, + 'geom': {'required': True}, } diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index b189919008..c3a9e9622f 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -141,6 +141,7 @@ class TestGeotrekTrekParser(GeotrekTrekParser): 'practice': {'create': True}, 'accessibilities': {'create': True}, 'networks': {'create': True}, + 'geom': {'required': True}, } @@ -149,6 +150,7 @@ class TestGeotrekPOIParser(GeotrekPOIParser): field_options = { 'type': {'create': True, }, + 'geom': {'required': True}, } @@ -157,6 +159,7 @@ class TestGeotrekServiceParser(GeotrekServiceParser): field_options = { 'type': {'create': True, }, + 'geom': {'required': True}, } From 4aa97c407246f67f3ad57877f2e2c06685a65826 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 1 Aug 2022 15:46:28 +0200 Subject: [PATCH 018/116] Fix linting --- geotrek/common/parsers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 5a9683fa12..63ec1942d6 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -18,7 +18,6 @@ from urllib.parse import urlparse from django.contrib.gis.geos import GEOSGeometry, WKBWriter -from django.core.management import call_command from django.db import models, connection from django.db.utils import DatabaseError from django.contrib.auth import get_user_model @@ -917,8 +916,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and - not f.name == 'id') + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') self.m2m_fields = { f.name: f.name for f in self.model._meta.many_to_many From 9538c4bf081cb4c55a1ea57e5e9be6c67096ff5c Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 1 Aug 2022 16:30:02 +0200 Subject: [PATCH 019/116] Add tests geom parsers api v2 --- geotrek/infrastructure/tests/test_parsers.py | 3 ++- geotrek/signage/tests/test_parsers.py | 3 ++- geotrek/tourism/parsers.py | 2 +- geotrek/tourism/tests/test_parsers.py | 6 ++++++ geotrek/trekking/tests/test_parsers.py | 6 ++++++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/geotrek/infrastructure/tests/test_parsers.py b/geotrek/infrastructure/tests/test_parsers.py index 84e209a6cf..b252d8287c 100644 --- a/geotrek/infrastructure/tests/test_parsers.py +++ b/geotrek/infrastructure/tests/test_parsers.py @@ -51,4 +51,5 @@ def mocked_json(): infrastructure = Infrastructure.objects.all().first() self.assertEqual(str(infrastructure.name), 'Table pic-nique') self.assertEqual(str(infrastructure.type), 'Table') - self.assertEqual(str(infrastructure.geom.ewkt), 'SRID=2154;POINT (565008.6693905985 6188246.533542466)') + self.assertAlmostEqual(infrastructure.geom.x, 565008.6693905985, places=5) + self.assertAlmostEqual(infrastructure.geom.y, 6188246.533542466, places=5) diff --git a/geotrek/signage/tests/test_parsers.py b/geotrek/signage/tests/test_parsers.py index e1f6a4bf75..ef8816b39c 100644 --- a/geotrek/signage/tests/test_parsers.py +++ b/geotrek/signage/tests/test_parsers.py @@ -53,4 +53,5 @@ def mocked_json(): self.assertEqual(str(signage.name), 'test gard') self.assertEqual(str(signage.type), 'Limite Cœur') self.assertEqual(str(signage.sealing), 'Socle béton') - self.assertEqual(str(signage.geom.ewkt), 'SRID=2154;POINT (572941.1308660918 6189000.155980503)') + self.assertAlmostEqual(signage.geom.x, 572941.1308660918, places=5) + self.assertAlmostEqual(signage.geom.y, 6189000.155980503, places=5) diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 26f219abc1..5c1c3b9516 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -1025,7 +1025,7 @@ def filter_eid(self, src, val): def filter_geom(self, src, val): lat, lng = val - return Point(lat, lng, srid=settings.API_SRID).transform(settings.SRID, clone=True) + return Point(lng, lat, srid=settings.API_SRID).transform(settings.SRID, clone=True) def filter_type(self, src, val): return self.apply_filter('type', src, val["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE]) diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 4a15298a4f..5d7e95f2ee 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -713,6 +713,8 @@ def mocked_json(): touristic_content = TouristicContent.objects.all().first() self.assertEqual(str(touristic_content.category), 'Sorties') self.assertEqual(str(touristic_content.name), "Balad'âne") + self.assertAlmostEqual(touristic_content.geom.x, 568112.6362873032, places=5) + self.assertAlmostEqual(touristic_content.geom.y, 6196929.676669887, places=5) class TouristicEventGeotrekParserTests(TestCase): @@ -746,6 +748,8 @@ def mocked_json(): touristic_event = TouristicEvent.objects.all().first() self.assertEqual(str(touristic_event.type), 'Spectacle') self.assertEqual(str(touristic_event.name), "Autrefois le Couserans") + self.assertAlmostEqual(touristic_event.geom.x, 548907.5259389633, places=5) + self.assertAlmostEqual(touristic_event.geom.y, 6208918.713349126, places=5) class InformationDeskGeotrekParserTests(TestCase): @@ -778,3 +782,5 @@ def mocked_json(): information_desk = InformationDesk.objects.all().first() self.assertEqual(str(information_desk.type), "Relais d'information") self.assertEqual(str(information_desk.name), "Foo") + self.assertAlmostEqual(information_desk.geom.x, 573013.9272605104, places=5) + self.assertAlmostEqual(information_desk.geom.y, 6276967.321705549, places=5) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index c3a9e9622f..1eeb23b804 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -195,6 +195,8 @@ def mocked_json(): self.assertEqual(trek.name, "Loop of the pic of 3 lords") self.assertEqual(str(trek.difficulty), 'Very easy') self.assertEqual(str(trek.practice), 'Horse') + self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) + self.assertAlmostEqual(trek.geom[0][1], 6190964.893167565, places=5) @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') @@ -228,6 +230,8 @@ def mocked_json(): poi = POI.objects.all().first() self.assertEqual(poi.name, "Pic des Trois Seigneurs") self.assertEqual(str(poi.type), 'Sommet') + self.assertAlmostEqual(poi.geom.x, 572298.7056448072, places=5) + self.assertAlmostEqual(poi.geom.y, 6193580.839504813, places=5) @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') @@ -260,3 +264,5 @@ def mocked_json(): self.assertEqual(Service.objects.count(), 2) service = Service.objects.all().first() self.assertEqual(str(service.type), 'Eau potable') + self.assertAlmostEqual(service.geom.x, 572096.2266745908, places=5) + self.assertAlmostEqual(service.geom.y, 6192330.15779677, places=5) From e914fcc48d2f36b8bb03dc14b39c3977e83f2b7e Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 1 Aug 2022 17:38:19 +0200 Subject: [PATCH 020/116] Change mock response requests get information desk --- geotrek/tourism/tests/test_parsers.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 5d7e95f2ee..36c9a4f7fc 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -763,6 +763,7 @@ def setUpTestData(cls): def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['informationdesk.json', ] + self.mock_content_time = 0 def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -771,11 +772,19 @@ def mocked_json(): with open(filename, 'r') as f: return json.load(f) + def mocked_requests_get(*args, **kwargs): + response = requests.Response() + response.status_code = 200 + if self.mock_content_time > 0: + response._content = None + else: + response._content = b'' + self.mock_content_time += 1 + response.json = mocked_json + return response + # Mock GET - mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json - mocked_get.return_value.content = b'' - mocked_head.return_value.status_code = 200 + mocked_get.side_effect = mocked_requests_get call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekInformationDeskParser', verbosity=0) self.assertEqual(InformationDesk.objects.count(), 2) From a6dacf7ecbd67b229c3a2c9bbf2f8d99ab8ab74f Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 1 Aug 2022 19:29:03 +0200 Subject: [PATCH 021/116] Add children parser api v2 --- geotrek/common/parsers.py | 13 +- geotrek/trekking/parsers.py | 25 +- .../tests/data/geotrek_parser_v2/trek.json | 3360 ++++------------- geotrek/trekking/tests/test_parsers.py | 12 +- 4 files changed, 804 insertions(+), 2606 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 63ec1942d6..720f7c90ab 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -901,7 +901,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): url = None separator = None delete = True - eid = 'eid' + eid = 'uuid' constant_fields = {} url_categories = {} replace_fields = {} @@ -913,9 +913,13 @@ class GeotrekParser(AttachmentParserMixin, Parser): field_options = { 'geom': {'required': True}, } + bbox = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) + self.bbox.srid = settings.SRID + self.bbox.transform(4326) # WGS84 self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') self.m2m_fields = { f.name: f.name @@ -929,7 +933,7 @@ def __init__(self, *args, **kwargs): self.translated_fields = [field for field in get_translated_fields(self.model)] for category in self.url_categories.keys(): route = self.url_categories[category] - response = self.request_or_retry(f"{self.url}{route}", ) # params=params) + response = self.request_or_retry(f"{self.url}{route}") self.field_options.setdefault(category, {}) if self.categories_keys_api_v2.get(category): self.field_options[category]["mapping"] = {} @@ -972,12 +976,9 @@ def filter_geom(self, src, val): return geom def next_row(self): - bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) - bbox.srid = settings.SRID - bbox.transform(4326) # WGS84 while self.next_url: params = { - 'in_bbox': ','.join([str(coord) for coord in bbox.extent]), + 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), } response = self.request_or_retry(self.next_url, params=params) self.root = response.json() diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 7b8a9b2bcf..9ae5920194 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -4,7 +4,7 @@ from django.utils.translation import gettext as _ from geotrek.common.parsers import ShapeParser, AttachmentParserMixin, GeotrekParser -from geotrek.trekking.models import POI, Service, Trek +from geotrek.trekking.models import OrderedTrekChild, POI, Service, Trek class DurationParserMixin: @@ -114,6 +114,29 @@ def filter_points_reference(self, src, val): geom = GEOSGeometry(json.dumps(val)) return geom.transform(settings.SRID, clone=True) + def end(self): + super().end() + self.next_url = f"{self.url}/api/v2/trek" + params = { + 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), + 'fields': ['children', 'id', 'uuid'] + } + response = self.request_or_retry(f"{self.next_url}", params=params) + results = response.json()['results'] + final_children = {} + for result in results: + final_children[result['id']] = {'uuid': result['uuid'], 'children': result['children']} + + for key, value in final_children.items(): + if value['children']: + trek_parent_instance = Trek.objects.get(eid=value['uuid']) + order = 0 + for child in value['children']: + trek_child_uuid = final_children.get(child)['uuid'] + trek_child_instance = Trek.objects.get(eid=trek_child_uuid) + OrderedTrekChild.objects.create(parent=trek_parent_instance, child=trek_child_instance, order=order) + order += 1 + class GeotrekServiceParser(GeotrekParser): url = None diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json index 6e49e92eae..553595d5aa 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -1,5 +1,5 @@ { - "count": 2, + "count": 5, "next": null, "previous": null, "results": [ @@ -63,7 +63,7 @@ "es": "", "it": "" }, - "altimetric_profile": "https://bidon.fr/api/v2/trek/2/profile/", + "altimetric_profile": "https://foo.fr/api/v2/trek/2/profile/", "ambiance": { "fr": "

Le nom de ce pic est issu de la légende selon laquelle les trois seigneurs des vallées de Massat, Vicdessos et Rabat-les-Trois-Seigneurs, se rencontraient sur la dalle plate en son sommet afin de débattre des droits des différentes vallées qu'ils administraient.

\r\n

\r\n

À partir du xviie siècle, de grandes caravanes d'ânes et de mulets transportaient le charbon de bois entre les forêts du Couserans et les forges à la catalane de la vallée de Rabat via le col de la Pourtanelle sur l'épaulement nord du pic.

\r\n

\r\n

Au xixe siècle, des porteurs de glace venaient y chercher leur butin sur le flanc nord du pic au glacier d'Ambans pour le transporter ensuite vers Toulouse. Ce glacier a totalement disparu au début du xxie siècle.

\r\n

\r\n

\r\n

Source Wikipedia

\r\n

", "en": "Ambiance en", @@ -174,1963 +174,68 @@ 1.4117038, 42.8065905, 1516.0 - ], - [ - 1.411729, - 42.8067836, - 1516.0 - ], - [ - 1.4117542, - 42.8069768, - 1518.0 - ], - [ - 1.4117794, - 42.80717, - 1520.0 - ], - [ - 1.4118407, - 42.8073758, - 1523.0 - ], - [ - 1.4119021, - 42.8075816, - 1528.0 - ], - [ - 1.4119634, - 42.8077874, - 1536.0 - ], - [ - 1.4120248, - 42.8079932, - 1544.0 - ], - [ - 1.4120861, - 42.808199, - 1553.0 - ], - [ - 1.412187, - 42.8083653, - 1563.0 - ], - [ - 1.4122878, - 42.8085316, - 1571.0 - ], - [ - 1.4123887, - 42.8086978, - 1578.0 - ], - [ - 1.4124895, - 42.8088641, - 1584.0 - ], - [ - 1.4125904, - 42.8090304, - 1591.0 - ], - [ - 1.4127306, - 42.8092155, - 1597.0 - ], - [ - 1.4128708, - 42.8094005, - 1603.0 - ], - [ - 1.4130111, - 42.8095856, - 1609.0 - ], - [ - 1.4131513, - 42.8097706, - 1615.0 - ], - [ - 1.4133876, - 42.8098607, - 1620.0 - ], - [ - 1.4136239, - 42.8099507, - 1625.0 - ], - [ - 1.4137589, - 42.8101307, - 1630.0 - ], - [ - 1.4138939, - 42.8103107, - 1635.0 - ], - [ - 1.4138545, - 42.810502, - 1641.0 - ], - [ - 1.4138152, - 42.8106933, - 1647.0 - ], - [ - 1.4137758, - 42.8108846, - 1652.0 - ], - [ - 1.4137364, - 42.8110759, - 1659.0 - ], - [ - 1.4137476, - 42.8112784, - 1665.0 - ], - [ - 1.4137589, - 42.8114809, - 1671.0 - ], - [ - 1.4137308, - 42.8116778, - 1679.0 - ], - [ - 1.4137027, - 42.8118747, - 1688.0 - ], - [ - 1.4136745, - 42.8120717, - 1695.0 - ], - [ - 1.4136464, - 42.8122686, - 1703.0 - ], - [ - 1.4136745, - 42.8124542, - 1712.0 - ], - [ - 1.4137026, - 42.8126399, - 1719.0 - ], - [ - 1.4137308, - 42.8128255, - 1726.0 - ], - [ - 1.4137589, - 42.8130112, - 1735.0 - ], - [ - 1.413851, - 42.8132175, - 1746.0 - ], - [ - 1.4139431, - 42.8134237, - 1755.0 - ], - [ - 1.4140352, - 42.81363, - 1766.0 - ], - [ - 1.4141274, - 42.8138362, - 1777.0 - ], - [ - 1.4142195, - 42.8140425, - 1787.0 - ], - [ - 1.4143116, - 42.8142487, - 1796.0 - ], - [ - 1.4144037, - 42.814455, - 1803.0 - ], - [ - 1.4146176, - 42.8145887, - 1811.0 - ], - [ - 1.4148315, - 42.8147224, - 1818.0 - ], - [ - 1.4150454, - 42.8148562, - 1825.0 - ], - [ - 1.4152593, - 42.8149899, - 1834.0 - ], - [ - 1.4154732, - 42.8151236, - 1844.0 - ], - [ - 1.4156871, - 42.8152573, - 1855.0 - ], - [ - 1.4158095, - 42.8154518, - 1865.0 - ], - [ - 1.4159319, - 42.8156464, - 1875.0 - ], - [ - 1.4160543, - 42.8158409, - 1887.0 - ], - [ - 1.4161767, - 42.8160354, - 1896.0 - ], - [ - 1.4161243, - 42.8162275, - 1905.0 - ], - [ - 1.416072, - 42.8164195, - 1912.0 - ], - [ - 1.4160196, - 42.8166116, - 1918.0 - ], - [ - 1.4158625, - 42.8167351, - 1921.0 - ], - [ - 1.4157054, - 42.8168586, - 1923.0 - ], - [ - 1.4155483, - 42.816982, - 1922.0 - ], - [ - 1.4153912, - 42.8171055, - 1923.0 - ], - [ - 1.415288, - 42.8172932, - 1926.0 - ], - [ - 1.4151847, - 42.8174808, - 1929.0 - ], - [ - 1.4150815, - 42.8176685, - 1934.0 - ], - [ - 1.4149782, - 42.8178561, - 1939.0 - ], - [ - 1.414875, - 42.8180438, - 1941.0 - ], - [ - 1.4150321, - 42.8182149, - 1942.0 - ], - [ - 1.4151892, - 42.818386, - 1941.0 - ], - [ - 1.4153464, - 42.8185572, - 1940.0 - ], - [ - 1.4155035, - 42.8187283, - 1940.0 - ], - [ - 1.4156606, - 42.8188994, - 1942.0 - ], - [ - 1.4158139, - 42.8190861, - 1943.0 - ], - [ - 1.4159673, - 42.8192727, - 1944.0 - ], - [ - 1.4161206, - 42.8194594, - 1945.0 - ], - [ - 1.4162739, - 42.819646, - 1946.0 - ], - [ - 1.4164272, - 42.8198327, - 1947.0 - ], - [ - 1.4165806, - 42.8200193, - 1952.0 - ], - [ - 1.4167339, - 42.820206, - 1957.0 - ], - [ - 1.4169439, - 42.8203343, - 1963.0 - ], - [ - 1.4171539, - 42.8204627, - 1969.0 - ], - [ - 1.4173639, - 42.820591, - 1977.0 - ], - [ - 1.4175739, - 42.8207193, - 1981.0 - ], - [ - 1.4177255, - 42.820906, - 1983.0 - ], - [ - 1.4178772, - 42.8210926, - 1984.0 - ], - [ - 1.4180288, - 42.8212793, - 1982.0 - ], - [ - 1.4181805, - 42.8214659, - 1979.0 - ], - [ - 1.4183205, - 42.8216246, - 1976.0 - ], - [ - 1.4184605, - 42.8217832, - 1973.0 - ], - [ - 1.4186005, - 42.8219419, - 1970.0 - ], - [ - 1.4187405, - 42.8221005, - 1968.0 - ], - [ - 1.4188805, - 42.8222592, - 1965.0 - ], - [ - 1.4189738, - 42.8224459, - 1963.0 - ], - [ - 1.4190671, - 42.8226325, - 1961.0 - ], - [ - 1.4191604, - 42.8228192, - 1960.0 - ], - [ - 1.4192538, - 42.8230059, - 1959.0 - ], - [ - 1.4193471, - 42.8231925, - 1959.0 - ], - [ - 1.4194404, - 42.8233792, - 1960.0 - ], - [ - 1.4195937, - 42.8235459, - 1961.0 - ], - [ - 1.4197471, - 42.8237125, - 1962.0 - ], - [ - 1.4199004, - 42.8238792, - 1963.0 - ], - [ - 1.4200537, - 42.8240458, - 1965.0 - ], - [ - 1.420207, - 42.8242125, - 1966.0 - ], + ] + ] + }, + "gpx": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.gpx", + "information_desks": [ + 2, + 1, + 3 + ], + "kml": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.kml", + "labels": [ + 1, + 2, + 3 + ], + "length_2d": 7606.3, + "length_3d": 7811.9, + "max_elevation": 2042, + "min_elevation": 1358, + "name": { + "fr": "Boucle du Pic des Trois Seigneurs", + "en": "Loop of the pic of 3 lords", + "es": "", + "it": "" + }, + "networks": [ + 2 + ], + "next": {}, + "parents": [], + "parking_location": [ + 1.412816, + 42.8063269 + ], + "pdf": { + "fr": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "en": "https://foo.fr/api/en/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "es": "https://foo.fr/api/es/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", + "it": "https://foo.fr/api/it/treks/2/boucle-du-pic-des-trois-seigneurs.pdf" + }, + "points_reference": { + "type": "MultiPoint", + "coordinates": [ [ - 1.4203604, - 42.8243791, - 1969.0 + 1.411417008494028, + 42.81070319713575 ], [ - 1.4205137, - 42.8245458, - 1973.0 + 1.415279389475474, + 42.81693648180148 ], [ - 1.4207678, - 42.8246288, - 1977.0 + 1.403348923777236, + 42.825246550724195 ], [ - 1.4210218, - 42.8247117, - 1981.0 + 1.428325654123916, + 42.8286458024494 ], [ - 1.4212759, - 42.8247947, - 1985.0 - ], - [ - 1.42153, - 42.8248776, - 1985.0 - ], - [ - 1.421784, - 42.8249606, - 1985.0 - ], - [ - 1.4220381, - 42.8250435, - 1985.0 - ], - [ - 1.4222922, - 42.8251265, - 1986.0 - ], - [ - 1.4225462, - 42.8252094, - 1989.0 - ], - [ - 1.4228003, - 42.8252924, - 1992.0 - ], - [ - 1.423047, - 42.8253924, - 1998.0 - ], - [ - 1.4232936, - 42.8254924, - 2003.0 - ], - [ - 1.4235403, - 42.8255924, - 2005.0 - ], - [ - 1.4237869, - 42.8256924, - 2006.0 - ], - [ - 1.4240336, - 42.8257924, - 2008.0 - ], - [ - 1.4242802, - 42.8258924, - 2009.0 - ], - [ - 1.4245269, - 42.8259924, - 2012.0 - ], - [ - 1.4248011, - 42.8260844, - 2018.0 - ], - [ - 1.4250753, - 42.8261765, - 2022.0 - ], - [ - 1.4253494, - 42.8262685, - 2023.0 - ], - [ - 1.4256236, - 42.8263606, - 2021.0 - ], - [ - 1.4258978, - 42.8264526, - 2019.0 - ], - [ - 1.426172, - 42.8265446, - 2016.0 - ], - [ - 1.4264462, - 42.8266367, - 2013.0 - ], - [ - 1.4267203, - 42.8267287, - 2013.0 - ], - [ - 1.4269945, - 42.8268208, - 2015.0 - ], - [ - 1.4272687, - 42.8269128, - 2016.0 - ], - [ - 1.4275282, - 42.827026, - 2017.0 - ], - [ - 1.4277877, - 42.8271393, - 2018.0 - ], - [ - 1.4280472, - 42.8272525, - 2018.0 - ], - [ - 1.4283067, - 42.8273658, - 2018.0 - ], - [ - 1.4285662, - 42.827479, - 2018.0 - ], - [ - 1.4288257, - 42.8275923, - 2019.0 - ], - [ - 1.4290852, - 42.8277055, - 2019.0 - ], - [ - 1.429323, - 42.8278244, - 2019.0 - ], - [ - 1.4295608, - 42.8279433, - 2019.0 - ], - [ - 1.4297986, - 42.8280622, - 2019.0 - ], - [ - 1.4300364, - 42.8281811, - 2018.0 - ], - [ - 1.4302742, - 42.8283, - 2016.0 - ], - [ - 1.430512, - 42.8284189, - 2014.0 - ], - [ - 1.4307499, - 42.8285378, - 2013.0 - ], - [ - 1.4309877, - 42.8286567, - 2013.0 - ], - [ - 1.4312255, - 42.8287756, - 2016.0 - ], - [ - 1.4314633, - 42.8288945, - 2021.0 - ], - [ - 1.431711, - 42.8289441, - 2028.0 - ], - [ - 1.4319587, - 42.8289936, - 2033.0 - ], - [ - 1.4321073, - 42.8289606, - 2038.0 - ], - [ - 1.4322559, - 42.8289275, - 2041.0 - ], - [ - 1.4324871, - 42.8289606, - 2042.0 - ], - [ - 1.4327182, - 42.8289936, - 2041.0 - ], - [ - 1.4327183, - 42.8289936, - 2041.0 - ], - [ - 1.4329385, - 42.8290266, - 2040.0 - ], - [ - 1.4331587, - 42.8290596, - 2037.0 - ], - [ - 1.4333789, - 42.8290926, - 2035.0 - ], - [ - 1.4335683, - 42.8291781, - 2031.0 - ], - [ - 1.4337577, - 42.8292636, - 2026.0 - ], - [ - 1.4340382, - 42.8293241, - 2020.0 - ], - [ - 1.4339593, - 42.8291741, - 2015.0 - ], - [ - 1.4339948, - 42.8290636, - 2009.0 - ], - [ - 1.4340303, - 42.828953, - 2002.0 - ], - [ - 1.4342435, - 42.8287952, - 1996.0 - ], - [ - 1.4342988, - 42.8286729, - 1991.0 - ], - [ - 1.434354, - 42.8285505, - 1987.0 - ], - [ - 1.4343974, - 42.8283649, - 1984.0 - ], - [ - 1.4344408, - 42.8281794, - 1982.0 - ], - [ - 1.4345277, - 42.8280216, - 1982.0 - ], - [ - 1.4346422, - 42.8279387, - 1982.0 - ], - [ - 1.4347566, - 42.8278558, - 1982.0 - ], - [ - 1.4347568, - 42.8278555, - 1981.0 - ], - [ - 1.4348513, - 42.8277295, - 1980.0 - ], - [ - 1.43486, - 42.8275735, - 1978.0 - ], - [ - 1.4348686, - 42.8274176, - 1975.0 - ], - [ - 1.4348934, - 42.8272506, - 1972.0 - ], - [ - 1.4349181, - 42.8270837, - 1968.0 - ], - [ - 1.4350047, - 42.8269662, - 1963.0 - ], - [ - 1.4350912, - 42.8268487, - 1958.0 - ], - [ - 1.4352891, - 42.8267683, - 1953.0 - ], - [ - 1.435487, - 42.8266879, - 1949.0 - ], - [ - 1.4355942, - 42.8265189, - 1945.0 - ], - [ - 1.4357014, - 42.8263498, - 1943.0 - ], - [ - 1.4358086, - 42.8261808, - 1940.0 - ], - [ - 1.435905, - 42.8259814, - 1937.0 - ], - [ - 1.4360013, - 42.8257819, - 1933.0 - ], - [ - 1.4360337, - 42.8255716, - 1927.0 - ], - [ - 1.4359447, - 42.8253855, - 1919.0 - ], - [ - 1.4358557, - 42.825244, - 1912.0 - ], - [ - 1.4357667, - 42.8251024, - 1905.0 - ], - [ - 1.4356964, - 42.8249714, - 1898.0 - ], - [ - 1.4356261, - 42.8248404, - 1892.0 - ], - [ - 1.4354401, - 42.8247067, - 1890.0 - ], - [ - 1.4352541, - 42.824573, - 1887.0 - ], - [ - 1.435068, - 42.8244392, - 1884.0 - ], - [ - 1.434882, - 42.8243055, - 1880.0 - ], - [ - 1.434696, - 42.8241718, - 1877.0 - ], - [ - 1.4345507, - 42.8240207, - 1870.0 - ], - [ - 1.4344053, - 42.8238695, - 1863.0 - ], - [ - 1.43426, - 42.8237184, - 1856.0 - ], - [ - 1.4341146, - 42.8235672, - 1848.0 - ], - [ - 1.4339693, - 42.8234161, - 1840.0 - ], - [ - 1.4338046, - 42.8233192, - 1833.0 - ], - [ - 1.4336398, - 42.8232223, - 1826.0 - ], - [ - 1.4334751, - 42.8231254, - 1821.0 - ], - [ - 1.4334557, - 42.822951, - 1816.0 - ], - [ - 1.4334364, - 42.8227766, - 1813.0 - ], - [ - 1.433417, - 42.8226022, - 1809.0 - ], - [ - 1.433417, - 42.8224181, - 1804.0 - ], - [ - 1.433417, - 42.822234, - 1798.0 - ], - [ - 1.433417, - 42.8220499, - 1791.0 - ], - [ - 1.4336011, - 42.8219142, - 1784.0 - ], - [ - 1.4337852, - 42.8217786, - 1776.0 - ], - [ - 1.4339693, - 42.8216429, - 1770.0 - ], - [ - 1.4341728, - 42.8215092, - 1765.0 - ], - [ - 1.4343762, - 42.8213755, - 1760.0 - ], - [ - 1.4345797, - 42.8212417, - 1757.0 - ], - [ - 1.4347831, - 42.821108, - 1754.0 - ], - [ - 1.4349866, - 42.8209743, - 1753.0 - ], - [ - 1.4351494, - 42.8207999, - 1752.0 - ], - [ - 1.4353122, - 42.8206255, - 1752.0 - ], - [ - 1.435475, - 42.8204511, - 1752.0 - ], - [ - 1.4356378, - 42.8202767, - 1752.0 - ], - [ - 1.4358006, - 42.8201023, - 1751.0 - ], - [ - 1.4359895, - 42.819986, - 1750.0 - ], - [ - 1.4361785, - 42.8198698, - 1749.0 - ], - [ - 1.4363674, - 42.8197535, - 1746.0 - ], - [ - 1.4365563, - 42.8196372, - 1742.0 - ], - [ - 1.4366871, - 42.8195064, - 1738.0 - ], - [ - 1.4368179, - 42.8193756, - 1733.0 - ], - [ - 1.4370824, - 42.8193016, - 1730.0 - ], - [ - 1.4373469, - 42.8192275, - 1727.0 - ], - [ - 1.4376114, - 42.8191535, - 1726.0 - ], - [ - 1.4378396, - 42.8191193, - 1724.0 - ], - [ - 1.4380677, - 42.819085, - 1723.0 - ], - [ - 1.4380677, - 42.8189481, - 1720.0 - ], - [ - 1.4380677, - 42.8188112, - 1717.0 - ], - [ - 1.4381019, - 42.8186287, - 1713.0 - ], - [ - 1.4381361, - 42.8184462, - 1709.0 - ], - [ - 1.4381475, - 42.8182294, - 1706.0 - ], - [ - 1.4381589, - 42.8180127, - 1704.0 - ], - [ - 1.4382046, - 42.8178644, - 1704.0 - ], - [ - 1.4382502, - 42.8177161, - 1705.0 - ], - [ - 1.4382616, - 42.8175335, - 1706.0 - ], - [ - 1.438273, - 42.817351, - 1707.0 - ], - [ - 1.4382046, - 42.8171457, - 1706.0 - ], - [ - 1.4381361, - 42.8169403, - 1703.0 - ], - [ - 1.4380677, - 42.816735, - 1698.0 - ], - [ - 1.4380312, - 42.816557, - 1693.0 - ], - [ - 1.4379947, - 42.8163791, - 1687.0 - ], - [ - 1.4379582, - 42.8162011, - 1680.0 - ], - [ - 1.4379217, - 42.8160232, - 1672.0 - ], - [ - 1.4378852, - 42.8158452, - 1666.0 - ], - [ - 1.4378681, - 42.8156398, - 1660.0 - ], - [ - 1.4378509, - 42.8154345, - 1653.0 - ], - [ - 1.4378338, - 42.8152291, - 1647.0 - ], - [ - 1.4378167, - 42.8150238, - 1641.0 - ], - [ - 1.4378015, - 42.8148337, - 1636.0 - ], - [ - 1.4377863, - 42.8146435, - 1630.0 - ], - [ - 1.4377711, - 42.8144534, - 1624.0 - ], - [ - 1.4377863, - 42.8142785, - 1618.0 - ], - [ - 1.4378015, - 42.8141035, - 1612.0 - ], - [ - 1.4378167, - 42.8139286, - 1605.0 - ], - [ - 1.4378738, - 42.813746, - 1600.0 - ], - [ - 1.4379308, - 42.8135635, - 1596.0 - ], - [ - 1.4380563, - 42.8134951, - 1592.0 - ], - [ - 1.4381818, - 42.8134266, - 1588.0 - ], - [ - 1.438022, - 42.8132669, - 1582.0 - ], - [ - 1.4378623, - 42.8131072, - 1574.0 - ], - [ - 1.4376969, - 42.8129874, - 1564.0 - ], - [ - 1.4375315, - 42.8128677, - 1552.0 - ], - [ - 1.4373661, - 42.8127479, - 1540.0 - ], - [ - 1.4372007, - 42.8126281, - 1528.0 - ], - [ - 1.4370258, - 42.8125216, - 1517.0 - ], - [ - 1.4368508, - 42.8124152, - 1507.0 - ], - [ - 1.4366759, - 42.8123087, - 1497.0 - ], - [ - 1.4364788, - 42.8122185, - 1488.0 - ], - [ - 1.4362818, - 42.8121283, - 1480.0 - ], - [ - 1.4360659, - 42.8122342, - 1473.0 - ], - [ - 1.4358459, - 42.8121406, - 1466.0 - ], - [ - 1.435626, - 42.8120469, - 1460.0 - ], - [ - 1.4354631, - 42.8121542, - 1455.0 - ], - [ - 1.4353001, - 42.8122614, - 1450.0 - ], - [ - 1.4351372, - 42.8123687, - 1445.0 - ], - [ - 1.4348774, - 42.8123797, - 1441.0 - ], - [ - 1.4346176, - 42.8123907, - 1437.0 - ], - [ - 1.4343578, - 42.8124017, - 1435.0 - ], - [ - 1.4342931, - 42.8123409, - 1433.0 - ], - [ - 1.4343238, - 42.8122299, - 1431.0 - ], - [ - 1.4343544, - 42.812119, - 1430.0 - ], - [ - 1.4343131, - 42.8120769, - 1428.0 - ], - [ - 1.434085, - 42.8120582, - 1426.0 - ], - [ - 1.4339295, - 42.81192, - 1423.0 - ], - [ - 1.4337741, - 42.8117817, - 1421.0 - ], - [ - 1.4335812, - 42.8116734, - 1419.0 - ], - [ - 1.4333474, - 42.8116359, - 1418.0 - ], - [ - 1.4332695, - 42.8116497, - 1416.0 - ], - [ - 1.4331915, - 42.8116852, - 1415.0 - ], - [ - 1.4330491, - 42.8115945, - 1414.0 - ], - [ - 1.4328631, - 42.8115593, - 1412.0 - ], - [ - 1.4326771, - 42.8115241, - 1409.0 - ], - [ - 1.4325105, - 42.8114348, - 1407.0 - ], - [ - 1.4323439, - 42.8113455, - 1403.0 - ], - [ - 1.4321773, - 42.8112562, - 1400.0 - ], - [ - 1.4319525, - 42.8111998, - 1397.0 - ], - [ - 1.4317277, - 42.8111435, - 1396.0 - ], - [ - 1.431503, - 42.8110871, - 1395.0 - ], - [ - 1.4312782, - 42.8110307, - 1395.0 - ], - [ - 1.4310659, - 42.8110366, - 1396.0 - ], - [ - 1.4309987, - 42.8109893, - 1397.0 - ], - [ - 1.4307793, - 42.8109726, - 1397.0 - ], - [ - 1.43056, - 42.8109559, - 1398.0 - ], - [ - 1.4304317, - 42.8109735, - 1399.0 - ], - [ - 1.4301818, - 42.8109045, - 1401.0 - ], - [ - 1.4300514, - 42.8108415, - 1403.0 - ], - [ - 1.4299211, - 42.8107784, - 1404.0 - ], - [ - 1.4296469, - 42.8107393, - 1405.0 - ], - [ - 1.4293728, - 42.8107001, - 1407.0 - ], - [ - 1.4291573, - 42.8106532, - 1408.0 - ], - [ - 1.4289417, - 42.8106064, - 1409.0 - ], - [ - 1.4287262, - 42.8105595, - 1410.0 - ], - [ - 1.4285199, - 42.8104738, - 1413.0 - ], - [ - 1.4283137, - 42.8103881, - 1415.0 - ], - [ - 1.4281968, - 42.8103029, - 1417.0 - ], - [ - 1.42808, - 42.8102177, - 1418.0 - ], - [ - 1.4279303, - 42.810098, - 1420.0 - ], - [ - 1.4278505, - 42.8098984, - 1420.0 - ], - [ - 1.4279127, - 42.8096897, - 1421.0 - ], - [ - 1.4279593, - 42.8095169, - 1422.0 - ], - [ - 1.4280058, - 42.8093441, - 1422.0 - ], - [ - 1.4280581, - 42.8092206, - 1423.0 - ], - [ - 1.4281104, - 42.809097, - 1422.0 - ], - [ - 1.4281494, - 42.8089847, - 1421.0 - ], - [ - 1.4281883, - 42.8088723, - 1419.0 - ], - [ - 1.428204, - 42.808672, - 1416.0 - ], - [ - 1.4282196, - 42.8084718, - 1411.0 - ], - [ - 1.4282381, - 42.808257, - 1405.0 - ], - [ - 1.4281994, - 42.8080872, - 1398.0 - ], - [ - 1.4281607, - 42.8079174, - 1391.0 - ], - [ - 1.4279773, - 42.8077578, - 1382.0 - ], - [ - 1.4277756, - 42.8076956, - 1375.0 - ], - [ - 1.427574, - 42.8076334, - 1369.0 - ], - [ - 1.4272872, - 42.8076402, - 1364.0 - ], - [ - 1.4270004, - 42.807647, - 1360.0 - ], - [ - 1.4267136, - 42.8076538, - 1358.0 - ], - [ - 1.4264724, - 42.807701, - 1358.0 - ], - [ - 1.4262313, - 42.8077482, - 1360.0 - ], - [ - 1.4259901, - 42.8077954, - 1364.0 - ], - [ - 1.4257177, - 42.8078459, - 1370.0 - ], - [ - 1.4254454, - 42.8078965, - 1376.0 - ], - [ - 1.425173, - 42.807947, - 1380.0 - ], - [ - 1.4250661, - 42.8079604, - 1383.0 - ], - [ - 1.4249892, - 42.8079537, - 1385.0 - ], - [ - 1.4248756, - 42.8079671, - 1386.0 - ], - [ - 1.4247887, - 42.8080072, - 1387.0 - ], - [ - 1.4247253, - 42.808074, - 1387.0 - ], - [ - 1.4247012, - 42.8081018, - 1388.0 - ], - [ - 1.4246277, - 42.8080722, - 1386.0 - ], - [ - 1.4245895, - 42.8079349, - 1383.0 - ], - [ - 1.4245509, - 42.8079029, - 1380.0 - ], - [ - 1.4244103, - 42.8079007, - 1376.0 - ], - [ - 1.4242745, - 42.807814, - 1372.0 - ], - [ - 1.4241388, - 42.8077273, - 1370.0 - ], - [ - 1.4240101, - 42.8075935, - 1371.0 - ], - [ - 1.4238485, - 42.8075043, - 1373.0 - ], - [ - 1.423687, - 42.8074151, - 1375.0 - ], - [ - 1.4234755, - 42.8073151, - 1377.0 - ], - [ - 1.423264, - 42.8072151, - 1378.0 - ], - [ - 1.4230455, - 42.807167, - 1378.0 - ], - [ - 1.4228271, - 42.8071189, - 1378.0 - ], - [ - 1.4226651, - 42.8071154, - 1378.0 - ], - [ - 1.4225032, - 42.8071118, - 1380.0 - ], - [ - 1.4222446, - 42.8070572, - 1383.0 - ], - [ - 1.4219817, - 42.8070129, - 1386.0 - ], - [ - 1.4218847, - 42.806984, - 1389.0 - ], - [ - 1.4218024, - 42.8069175, - 1392.0 - ], - [ - 1.4215877, - 42.8068009, - 1395.0 - ], - [ - 1.4214108, - 42.8066672, - 1398.0 - ], - [ - 1.4212519, - 42.8066211, - 1402.0 - ], - [ - 1.421093, - 42.8065749, - 1403.0 - ], - [ - 1.4209135, - 42.8065618, - 1405.0 - ] - ] - }, - "gpx": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.gpx", - "information_desks": [ - 2, - 1, - 3 - ], - "kml": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.kml", - "labels": [ - 1, - 2, - 3 - ], - "length_2d": 7606.3, - "length_3d": 7811.9, - "max_elevation": 2042, - "min_elevation": 1358, - "name": { - "fr": "Boucle du Pic des Trois Seigneurs", - "en": "Loop of the pic of 3 lords", - "es": "", - "it": "" - }, - "networks": [ - 2 - ], - "next": {}, - "parents": [], - "parking_location": [ - 1.412816, - 42.8063269 - ], - "pdf": { - "fr": "https://foo.fr/api/fr/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", - "en": "https://foo.fr/api/en/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", - "es": "https://foo.fr/api/es/treks/2/boucle-du-pic-des-trois-seigneurs.pdf", - "it": "https://foo.fr/api/it/treks/2/boucle-du-pic-des-trois-seigneurs.pdf" - }, - "points_reference": { - "type": "MultiPoint", - "coordinates": [ - [ - 1.411417008494028, - 42.81070319713575 - ], - [ - 1.415279389475474, - 42.81693648180148 - ], - [ - 1.403348923777236, - 42.825246550724195 - ], - [ - 1.428325654123916, - 42.8286458024494 - ], - [ - 1.437852860544806, - 42.81611800550653 + 1.437852860544806, + 42.81611800550653 ], [ 1.423759460449225, @@ -2376,687 +481,754 @@ ], [ 1.0852891, - 42.8640109, - 1061.0 - ], - [ - 1.0855466, - 42.8640926, - 1063.0 - ], - [ - 1.085804, - 42.8641744, - 1067.0 - ], - [ - 1.0860615, - 42.8642562, - 1072.0 - ], - [ - 1.086319, - 42.864338, - 1075.0 - ], - [ - 1.0865765, - 42.8644198, - 1076.0 - ], - [ - 1.086834, - 42.8645016, - 1079.0 - ], - [ - 1.0870915, - 42.8645833, - 1082.0 - ], - [ - 1.087349, - 42.8646651, - 1084.0 - ], - [ - 1.0876065, - 42.8647469, - 1086.0 - ], - [ - 1.087864, - 42.8648287, - 1088.0 - ], - [ - 1.0881215, - 42.8647186, - 1090.0 - ], - [ - 1.088379, - 42.8646085, - 1092.0 - ], - [ - 1.0886365, - 42.8644984, - 1094.0 - ], - [ - 1.088894, - 42.8643883, - 1096.0 - ], - [ - 1.0891515, - 42.8642782, - 1097.0 - ], - [ - 1.089409, - 42.8641681, - 1097.0 - ], - [ - 1.0896665, - 42.864058, - 1096.0 - ], - [ - 1.089924, - 42.863948, - 1094.0 - ], - [ - 1.0901814, - 42.8638379, - 1094.0 - ], - [ - 1.0904389, - 42.8637278, - 1096.0 - ], - [ - 1.0906964, - 42.8636177, - 1096.0 - ], - [ - 1.0909539, - 42.8635076, - 1098.0 - ], - [ - 1.0912114, - 42.8633975, - 1100.0 - ], - [ - 1.0914689, - 42.8632874, - 1103.0 - ], - [ - 1.0917264, - 42.8631773, - 1106.0 - ], - [ - 1.0919839, - 42.8630672, - 1110.0 - ], - [ - 1.0922168, - 42.8629324, - 1115.0 - ], - [ - 1.0924498, - 42.8627976, - 1119.0 - ], - [ - 1.0926828, - 42.8626627, - 1123.0 - ], - [ - 1.0929157, - 42.8625279, - 1127.0 - ], - [ - 1.0931487, - 42.8623931, - 1132.0 - ], - [ - 1.0933817, - 42.8622583, - 1136.0 - ], - [ - 1.0936146, - 42.8621235, - 1139.0 - ], - [ - 1.0937005, - 42.8619168, - 1145.0 - ], - [ - 1.0937863, - 42.86171, - 1152.0 - ], - [ - 1.0938721, - 42.8615033, - 1157.0 - ], - [ - 1.093958, - 42.8612966, - 1162.0 - ], - [ - 1.0940438, - 42.8610899, - 1167.0 - ], - [ - 1.0941296, - 42.8608832, - 1169.0 - ], - [ - 1.0942155, - 42.8606765, - 1169.0 - ], - [ - 1.0941255, - 42.8604667, - 1167.0 - ], - [ - 1.0940356, - 42.860257, - 1164.0 - ], - [ - 1.0939457, - 42.8600473, - 1161.0 - ], - [ - 1.0938558, - 42.8598376, - 1158.0 - ], - [ - 1.0937658, - 42.8596278, - 1155.0 - ], - [ - 1.0936759, - 42.8594181, - 1155.0 - ], - [ - 1.093586, - 42.8592084, - 1155.0 - ], - [ - 1.0934961, - 42.8589987, - 1157.0 - ], - [ - 1.0934062, - 42.858789, - 1157.0 - ], - [ - 1.0933162, - 42.8585792, - 1157.0 - ], - [ - 1.0932263, - 42.8583695, - 1154.0 - ], - [ - 1.0931364, - 42.8581598, - 1151.0 - ], - [ - 1.0930465, - 42.8579501, - 1147.0 - ], - [ - 1.0929566, - 42.8577403, - 1145.0 - ], - [ - 1.0928667, - 42.8575306, - 1144.0 - ], - [ - 1.0927767, - 42.8573209, - 1145.0 - ], - [ - 1.0926868, - 42.8571112, - 1148.0 - ], - [ - 1.0925969, - 42.8569014, - 1152.0 - ], - [ - 1.092507, - 42.8566917, - 1156.0 - ], - [ - 1.0924171, - 42.856482, - 1159.0 - ], - [ - 1.0923272, - 42.8562723, - 1161.0 - ], - [ - 1.0924676, - 42.8560892, - 1162.0 - ], - [ - 1.0926081, - 42.8559062, - 1163.0 - ], - [ - 1.0927485, - 42.8557232, - 1164.0 - ], - [ - 1.092889, - 42.8555401, - 1167.0 - ], - [ - 1.0930294, - 42.8553571, - 1171.0 - ], - [ - 1.0931699, - 42.855174, - 1176.0 - ], - [ - 1.0933103, - 42.854991, - 1183.0 - ], - [ - 1.0934508, - 42.8548079, - 1191.0 - ], - [ - 1.0935912, - 42.8546249, - 1197.0 - ], - [ - 1.0937317, - 42.8544419, - 1201.0 - ], - [ - 1.0938721, - 42.8542588, - 1205.0 - ], - [ - 1.0941094, - 42.854381, - 1202.0 - ], - [ - 1.0943467, - 42.8545031, - 1195.0 - ], - [ - 1.094584, - 42.8546253, - 1188.0 - ], - [ - 1.0948213, - 42.8547474, - 1181.0 - ], - [ - 1.0950586, - 42.8548695, - 1174.0 - ], - [ - 1.0952959, - 42.8549917, - 1169.0 - ], - [ - 1.0955332, - 42.8551138, - 1167.0 - ], - [ - 1.0957705, - 42.855236, - 1167.0 - ], - [ - 1.0960078, - 42.8553581, - 1167.0 - ], - [ - 1.0962451, - 42.8554802, - 1168.0 - ], - [ - 1.0964824, - 42.8556024, - 1172.0 - ], - [ - 1.0967197, - 42.8557245, - 1175.0 - ], - [ - 1.096957, - 42.8558466, - 1179.0 - ], - [ - 1.0971943, - 42.8559688, - 1185.0 - ], - [ - 1.0974316, - 42.8560909, - 1192.0 - ], - [ - 1.0976689, - 42.8562131, - 1201.0 - ], - [ - 1.0979062, - 42.8563352, - 1212.0 - ], - [ - 1.098044, - 42.8561365, - 1225.0 - ], - [ - 1.0981818, - 42.8559378, - 1240.0 - ], - [ - 1.0983196, - 42.8557391, - 1258.0 - ], - [ - 1.0984574, - 42.8555404, - 1273.0 - ], - [ - 1.0985952, - 42.8553417, - 1286.0 - ], - [ - 1.098733, - 42.855143, - 1300.0 - ], - [ - 1.0988707, - 42.8549443, - 1313.0 - ], - [ - 1.0990085, - 42.8547456, - 1325.0 - ], - [ - 1.0991463, - 42.8545469, - 1340.0 - ], - [ - 1.0992841, - 42.8543482, - 1359.0 - ], - [ - 1.0994219, - 42.8541495, - 1374.0 - ], - [ - 1.0995597, - 42.8539508, - 1388.0 - ], - [ - 1.0996975, - 42.8537521, - 1404.0 - ], - [ - 1.0998353, - 42.8535534, - 1418.0 - ], - [ - 1.0999731, - 42.8533547, - 1432.0 - ], - [ - 1.1001108, - 42.8531559, - 1449.0 - ], - [ - 1.1002486, - 42.8529572, - 1468.0 - ], - [ - 1.1003864, - 42.8527585, - 1485.0 - ], - [ - 1.1005242, - 42.8525598, - 1501.0 - ], - [ - 1.100662, - 42.8523611, - 1521.0 - ], - [ - 1.1007997, - 42.8521624, - 1541.0 - ], - [ - 1.1009375, - 42.8519637, - 1562.0 - ], - [ - 1.1010753, - 42.851765, - 1588.0 - ], - [ - 1.1012131, - 42.8515663, - 1615.0 - ], - [ - 1.1013509, - 42.8513676, - 1637.0 - ], - [ - 1.1014886, - 42.8511689, - 1656.0 - ], - [ - 1.1016264, - 42.8509702, - 1672.0 - ], - [ - 1.1017642, - 42.8507715, - 1683.0 - ], - [ - 1.1019019, - 42.8505728, - 1690.0 - ], - [ - 1.1020397, - 42.8503741, - 1696.0 - ], - [ - 1.1021775, - 42.8501754, - 1700.0 - ], + 42.8640109, + 1061.0 + ] + ] + }, + "gpx": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.gpx", + "information_desks": [], + "kml": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.kml", + "labels": [], + "length_2d": 3063.4, + "length_3d": 3245.4, + "max_elevation": 1711, + "min_elevation": 1057, + "name": { + "fr": "De Bethmale au col de la Core", + "en": "", + "es": "", + "it": "" + }, + "networks": [], + "next": { + "10445": null + }, + "parents": [ + 10445 + ], + "parking_location": null, + "pdf": { + "fr": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "en": "https://foo.fr/api/en/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "es": "https://foo.fr/api/es/treks/10443/de-bethmale-au-col-de-la-core.pdf", + "it": "https://foo.fr/api/it/treks/10443/de-bethmale-au-col-de-la-core.pdf" + }, + "points_reference": null, + "portal": [], + "practice": 4, + "ratings": [], + "ratings_description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "previous": { + "10445": 10441 + }, + "public_transport": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 3, + "second_external_id": null, + "source": [], + "structure": 3, + "themes": [], + "update_datetime": "2022-07-26T08:18:06.997582Z", + "url": "https://foo.fr/api/v2/trek/10443/", + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12", + "web_links": [] + }, + { + "id": 2849, + "access": { + "fr": "

Depuis le village d'Aulus-les-Bains.

", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [], + "accessibility_advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_covering": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_exposure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_slope": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_width": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advice": { + "fr": "Sentier parfois escarpé. Camping interdit. Feu interdit.", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "Dans le bas du village", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://foo.fr/api/v2/trek/2849/profile/", + "ambiance": { + "fr": "

Une des plus belles cascades des Pyrénées pour toute la famille ! C'est une cascade naturelle des Pyrénées située dans le Couserans en Ariège, à 1 380 m d'altitude. C'est une des plus belles et imposantes des Pyrénées. Le comte Henry Russell, célèbre pyrénéistes du XIXe siècle, la plaçait en tête de toutes ses rivales.

", + "en": "", + "es": "", + "it": "" + }, + "arrival": { + "fr": "Aulus-les-Bains (sud)", + "en": "", + "es": "", + "it": "" + }, + "ascent": 1130, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "Pierre Gouget", + "license": null, + "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/2849/cascade_dars_a_aulus-les-bains-cc-by-sa-pierre-gouget.jpg.400x0_q85.jpg", + "legend": "Cascade d'Ars au mois de Mai (CC-By-SA)", + "title": "", + "url": "https://foo.fr/media/paperclip/trekking_trek/2849/cascade_dars_a_aulus-les-bains-cc-by-sa-pierre-gouget.jpg", + "uuid": "7dfdc2cd-69f8-4ed6-9ed1-eab178390aca" + }, + { + "backend": "Attachment", + "type": "video", + "author": "Ariège Pyrénées", + "license": null, + "thumbnail": "", + "legend": "", + "title": "", + "url": "https://www.youtube.com/watch?v=O5fwnNceuks", + "uuid": "9fd25104-f758-4d8a-a7fa-241bc1b598dc" + }, + { + "backend": "", + "type": "image", + "author": "Tongatahu", + "license": null, + "thumbnail": "", + "legend": "Panneau (CC-By-SA)", + "title": "", + "url": "https://upload.wikimedia.org/wikipedia/commons/e/e3/Aulus-les-Bains_Panneau_d%27information_sur_le_chemin_de_cascade_d%27_ars.jpg", + "uuid": "ca7e611f-6656-42db-85f4-01f51131a5cf" + }, + { + "backend": "", + "type": "file", + "author": "Département d'Ariège", + "license": null, + "thumbnail": "", + "legend": "Activités pédestres", + "title": "activite_Pedestre", + "url": "https://foo.fr/media/paperclip/trekking_trek/2849/activite_pedestre.PDF", + "uuid": "d150f3f0-0c96-4f6d-8a54-3bbbe0dbe9d5" + }, + { + "backend": "", + "type": "image", + "author": "Mathieu MD", + "license": null, + "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/2849/14212843221_3cfb859ec4_o.jpg.400x0_q85.jpg", + "legend": "Cascade d'Ars - CC-By-SA", + "title": "14212843221_3cfb859ec4_o", + "url": "https://foo.fr/media/paperclip/trekking_trek/2849/14212843221_3cfb859ec4_o.jpg", + "uuid": "5ebbb31a-6a85-4377-9144-11130788a930" + } + ], + "attachments_accessibility": [], + "children": [], + "cities": [ + "09029" + ], + "create_datetime": "2013-12-13T10:51:27.121271Z", + "departure": { + "fr": "Aulus-les-Bains", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09029", + "departure_geom": [ + 1.342102928593241, + 42.78978331370928 + ], + "descent": -1033, + "description": { + "fr": "

On accède à la cascade en 90 mn de marche à partir de la D 8, au départ d'Aulus-les-Bains vers le col de Latrappe. Au premier virage en épingle (1) une piste forestière s'élève entre forêt et pâturage. On franchit ensuite la passerelle de l'Artigou (1 060 m - 2) puis on longe la rive droite de l'Ars jusqu'au pied de la chute d'eau (3).

", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "

Haute d'environ 246m, elle est a son apogée à la fonte des neiges.

", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 1, + "disabled_infrastructure": { + "fr": "

Aucune infrastructure spécifique

", + "en": "", + "es": "", + "it": "" + }, + "duration": 4.0, + "elevation_area_url": "https://foo.fr/api/v2/trek/2849/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/2849/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ - 1.1023153, - 42.8499767, - 1704.0 + 1.3421029, + 42.7897833, + 760.0 ], [ - 1.102453, - 42.849778, - 1707.0 - ], + 1.3422439, + 42.7896888, + 760.0 + ] + ] + }, + "gpx": "https://foo.fr/api/fr/treks/2849/decouverte-de-la-cascade-dars.gpx", + "information_desks": [], + "kml": "https://foo.fr/api/fr/treks/2849/decouverte-de-la-cascade-dars.kml", + "labels": [ + 1 + ], + "length_2d": 12218.2, + "length_3d": 12531.1, + "max_elevation": 1607, + "min_elevation": 760, + "name": { + "fr": "Découverte de la Cascade d'Ars", + "en": "", + "es": "", + "it": "" + }, + "networks": [ + 2 + ], + "next": {}, + "parents": [ + 4904 + ], + "parking_location": [ + 1.3386154, + 42.790173 + ], + "pdf": { + "fr": "https://foo.fr/api/fr/treks/2849/decouverte-de-la-cascade-dars.pdf", + "en": "https://foo.fr/api/en/treks/2849/decouverte-de-la-cascade-dars.pdf", + "es": "https://foo.fr/api/es/treks/2849/decouverte-de-la-cascade-dars.pdf", + "it": "https://foo.fr/api/it/treks/2849/decouverte-de-la-cascade-dars.pdf" + }, + "points_reference": { + "type": "MultiPoint", + "coordinates": [ [ - 1.1025908, - 42.8495793, - 1709.0 + 1.352026462554938, + 42.78377952689919 ], [ - 1.1027286, - 42.8493805, - 1710.0 + 1.349644660949709, + 42.77618839945826 ], [ - 1.1028663, - 42.8491818, - 1711.0 - ], + 1.351146697998048, + 42.76437462843613 + ] + ] + }, + "portal": [], + "practice": 4, + "ratings": [], + "ratings_description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "previous": {}, + "public_transport": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "published": { + "fr": true, + "en": true, + "es": false, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 2, + "second_external_id": null, + "source": [], + "structure": 1, + "themes": [ + 7 + ], + "update_datetime": "2022-05-18T10:32:44.170710Z", + "url": "https://foo.fr/api/v2/trek/2849/", + "uuid": "6761143f-9244-41d0-b1af-21114408f769", + "web_links": [] + }, + { + "id": 10439, + "access": { + "fr": "Bonnac Irazein", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [], + "accessibility_advice": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_covering": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_exposure": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_slope": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_width": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://foo.fr/api/v2/trek/10439/profile/", + "ambiance": { + "fr": "Je suis une bonne ambiance.", + "en": "", + "es": "", + "it": "" + }, + "arrival": { + "fr": "Ourjout", + "en": "", + "es": "", + "it": "" + }, + "ascent": 203, + "attachments": [ + { + "backend": "", + "type": "image", + "author": "Borvan53, CC-By-SA 4.0", + "license": null, + "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/10439/bocard_deylie_2013_01.JPG.400x0_q85.jpg", + "legend": "Vue du site du bocard d'Eylie", + "title": "", + "url": "https://foo.fr/media/paperclip/trekking_trek/10439/bocard_deylie_2013_01.JPG", + "uuid": "2ed7ebc0-39ed-482c-bae3-0b1182f27d3d" + }, + { + "backend": "", + "type": "image", + "author": "", + "license": null, + "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/10439/eeaio.jpeg.400x0_q85.jpg", + "legend": "", + "title": "éèàïô", + "url": "https://foo.fr/media/paperclip/trekking_trek/10439/eeaio.jpeg", + "uuid": "05c0c4ad-69bd-4b86-a8dc-3cbc74ecc4b6" + } + ], + "attachments_accessibility": [], + "children": [], + "cities": [ + "09290", + "09059", + "09317", + "09062" + ], + "create_datetime": "2019-07-23T09:09:53.090318Z", + "departure": { + "fr": "Eylé", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09290", + "departure_geom": [ + 0.9377712652787816, + 42.83789927043632 + ], + "descent": -564, + "description": { + "fr": "Dede est au bar.", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "Chapeau a fleur.", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 2, + "disabled_infrastructure": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "duration": 6.0, + "elevation_area_url": "https://foo.fr/api/v2/trek/10439/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/10439/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "geometry": { + "type": "LineString", + "coordinates": [ [ - 1.1030041, - 42.8489831, - 1711.0 + 0.9377713, + 42.8378993, + 912.0 ], [ - 1.1030671, - 42.8488923, - 1710.0 + 0.9377673, + 42.8379281, + 913.0 ] ] }, - "gpx": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.gpx", + "gpx": "https://foo.fr/api/fr/treks/10439/deyle-a-ourjout.gpx", "information_desks": [], - "kml": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.kml", + "kml": "https://foo.fr/api/fr/treks/10439/deyle-a-ourjout.kml", "labels": [], - "length_2d": 3063.4, - "length_3d": 3245.4, - "max_elevation": 1711, - "min_elevation": 1057, + "length_2d": 12856.3, + "length_3d": 12904.9, + "max_elevation": 927, + "min_elevation": 551, "name": { - "fr": "De Bethmale au col de la Core", + "fr": "Foo", "en": "", "es": "", "it": "" }, "networks": [], "next": { - "10445": null + "2": null, + "10445": 10441 }, "parents": [ + 2, 10445 ], "parking_location": null, "pdf": { - "fr": "https://foo.fr/api/fr/treks/10443/de-bethmale-au-col-de-la-core.pdf", - "en": "https://foo.fr/api/en/treks/10443/de-bethmale-au-col-de-la-core.pdf", - "es": "https://foo.fr/api/es/treks/10443/de-bethmale-au-col-de-la-core.pdf", - "it": "https://foo.fr/api/it/treks/10443/de-bethmale-au-col-de-la-core.pdf" + "fr": "https://foo.fr/api/fr/treks/10439/deyle-a-ourjout.pdf", + "en": "https://foo.fr/api/en/treks/10439/deyle-a-ourjout.pdf", + "es": "https://foo.fr/api/es/treks/10439/deyle-a-ourjout.pdf", + "it": "https://foo.fr/api/it/treks/10439/deyle-a-ourjout.pdf" }, "points_reference": null, "portal": [], "practice": 4, "ratings": [], "ratings_description": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "previous": { + "2": null, + "10445": null + }, + "public_transport": { "fr": "", "en": "", "es": "", "it": "" }, - "previous": { - "10445": 10441 + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 3, + "second_external_id": null, + "source": [], + "structure": 3, + "themes": [], + "update_datetime": "2021-05-17T13:54:07.091500Z", + "url": "https://foo.fr/api/v2/trek/10439/", + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9", + "web_links": [] + }, + { + "id": 8700, + "access": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [], + "accessibility_advice": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_covering": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_exposure": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_slope": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "accessibility_width": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://foo.fr/api/v2/trek/8700/profile/", + "ambiance": { + "fr": "Ambience", + "en": "", + "es": "", + "it": "" + }, + "arrival": { + "fr": "Étang", + "en": "", + "es": "", + "it": "" }, + "ascent": 1051, + "attachments": [], + "attachments_accessibility": [], + "children": [], + "cities": [ + "09322" + ], + "create_datetime": "2019-04-01T12:25:12.319886Z", + "departure": { + "fr": "Île aux moussures", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09322", + "departure_geom": [ + 1.28743696523673, + 42.75618193887717 + ], + "descent": -133, + "description": { + "fr": "Description", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "Chapeau", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 3, + "disabled_infrastructure": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "duration": 4.0, + "elevation_area_url": "https://foo.fr/api/v2/trek/8700/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/8700/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 1.287437, + 42.7561819, + 1010.0 + ], + [ + 1.2873813, + 42.756201, + 1007.0 + ] + ] + }, + "gpx": "https://foo.fr/api/fr/treks/8700/etang-dalet.gpx", + "information_desks": [], + "kml": "https://foo.fr/api/fr/treks/8700/etang-dalet.kml", + "labels": [ + 2 + ], + "length_2d": 4844.7, + "length_3d": 5073.4, + "max_elevation": 1936, + "min_elevation": 979, + "name": { + "fr": "Étang d'Alet", + "en": "", + "es": "", + "it": "" + }, + "networks": [], + "next": {}, + "parents": [], + "parking_location": null, + "pdf": { + "fr": "https://foo.fr/api/fr/treks/8700/etang-dalet.pdf", + "en": "https://foo.fr/api/en/treks/8700/etang-dalet.pdf", + "es": "https://foo.fr/api/es/treks/8700/etang-dalet.pdf", + "it": "https://foo.fr/api/it/treks/8700/etang-dalet.pdf" + }, + "points_reference": null, + "portal": [], + "practice": 4, + "ratings": [], + "ratings_description": { + "fr": "", + "en": null, + "es": null, + "it": null + }, + "previous": {}, "public_transport": { "fr": "", "en": "", @@ -3071,14 +1243,14 @@ }, "reservation_system": null, "reservation_id": "", - "route": 3, + "route": 2, "second_external_id": null, "source": [], "structure": 3, "themes": [], - "update_datetime": "2022-07-26T08:18:06.997582Z", - "url": "https://foo.fr/api/v2/trek/10443/", - "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12", + "update_datetime": "2021-04-28T14:45:50.070810Z", + "url": "https://foo.fr/api/v2/trek/8700/", + "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1", "web_links": [] } ] diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 1eeb23b804..d430f9d284 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -163,6 +163,7 @@ class TestGeotrekServiceParser(GeotrekServiceParser): } +@override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class TrekGeotrekParserTests(TestCase): @classmethod @@ -174,7 +175,7 @@ def setUpTestData(cls): def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', - 'trek_accessibility.json', 'trek_network.json', 'trek.json'] + 'trek_accessibility.json', 'trek_network.json', 'trek.json', 'trek_children.json'] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -190,13 +191,14 @@ def mocked_json(): mocked_head.return_value.status_code = 200 call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) - self.assertEqual(Trek.objects.count(), 2) + self.assertEqual(Trek.objects.count(), 5) trek = Trek.objects.all().first() - self.assertEqual(trek.name, "Loop of the pic of 3 lords") - self.assertEqual(str(trek.difficulty), 'Very easy') - self.assertEqual(str(trek.practice), 'Horse') + self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(str(trek.difficulty), 'Très facile') + self.assertEqual(str(trek.practice), 'Cheval') self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) self.assertAlmostEqual(trek.geom[0][1], 6190964.893167565, places=5) + self.assertEqual(trek.children.first().name, "Foo") @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') From 8d7df49e9e2d469a8e44e452413a070b9bcfc836 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 2 Aug 2022 10:08:24 +0200 Subject: [PATCH 022/116] Add children file trek --- .../data/geotrek_parser_v2/trek_children.json | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json new file mode 100644 index 0000000000..1bb157408d --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json @@ -0,0 +1,34 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 2, + "children": [ + 10439 + ], + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1" + }, + { + "id": 10443, + "children": [], + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12" + }, + { + "id": 2849, + "children": [], + "uuid": "6761143f-9244-41d0-b1af-21114408f769" + }, + { + "id": 10439, + "children": [], + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" + }, + { + "id": 8700, + "children": [], + "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1" + } + ] +} \ No newline at end of file From 6a849c3642d572bd7511c0b690bdcbbae8f65b7a Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 2 Aug 2022 10:44:54 +0200 Subject: [PATCH 023/116] Add filter uuid information desk --- geotrek/tourism/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 5c1c3b9516..3110663d80 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -997,7 +997,7 @@ class GeotrekInformationDeskParser(GeotrekParser): model = InformationDesk constant_fields = {} replace_fields = { - "eid": ["uuid", "id"], + "uuid": ["uuid", "id"], "geom": ["latitude", "longitude"], "photo": "photo_url" } @@ -1016,7 +1016,7 @@ def next_row(self): self.next_url = f"{self.url}/api/v2/informationdesk" return super().next_row() - def filter_eid(self, src, val): + def filter_uuid(self, src, val): uuid, id_iddesk = val final_value = uuid if not uuid: From 51d27e96ee1d796906621b73b9c086dfc9830d28 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 2 Aug 2022 11:40:47 +0200 Subject: [PATCH 024/116] Add test no content attachment download --- geotrek/common/tests/test_parsers.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 83eed5104d..924ee67e72 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -3,6 +3,7 @@ from shutil import rmtree from tempfile import mkdtemp from io import StringIO +import requests from requests import Response import urllib @@ -298,6 +299,25 @@ def test_attachment_download_fail(self, mocked_urlparse, mocked_get): self.assertEqual(mocked_get.call_count, 1) + @mock.patch('requests.get') + def test_attachment_no_content(self, mocked): + """ + It will always take the one without structure first + """ + def mocked_requests_get(*args, **kwargs): + response = requests.Response() + response.status_code = 200 + response._content = None + return response + + # Mock GET + mocked.side_effect = mocked_requests_get + structure = StructureFactory.create(name="Structure") + FileType.objects.create(type="Photographie", structure=structure) + filename = os.path.join(os.path.dirname(__file__), 'data', 'organism.xls') + call_command('import', 'geotrek.common.tests.test_parsers.AttachmentParser', filename, verbosity=0) + self.assertEqual(Attachment.objects.count(), 0) + class TourInSoftParserTests(TestCase): From 023d8f4e5691f10fb178f4eca024238828cae0b0 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 2 Aug 2022 12:26:33 +0200 Subject: [PATCH 025/116] Update test informaiton desk --- .../geotrek_parser_v2/informationdesk.json | 45 ++++++++++++++++++- geotrek/tourism/tests/test_parsers.py | 29 ++++++------ 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json index af7494757c..95c149b433 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json @@ -1,5 +1,5 @@ { - "count": 2, + "count": 3, "next": null, "previous": null, "results": [ @@ -70,7 +70,7 @@ "it": null }, "phone": null, - "photo_url": "", + "photo_url": "https://foo.fr/media/upload/office-seix.jpg.150x150_q85.jpg", "postal_code": "31300", "street": "52 rue Jacques Babinet", "type": { @@ -84,6 +84,47 @@ "pictogram": "https://foo.fr/media/upload/desktype-info.svg" }, "website": null + }, + { + "id": 3, + "accessibility": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "email": null, + "label_accessibility": null, + "latitude": null, + "longitude": null, + "municipality": null, + "name": { + "fr": "Test fso sans carte", + "en": null, + "es": null, + "it": null + }, + "phone": null, + "photo_url": "", + "postal_code": null, + "street": null, + "type": { + "id": 3, + "label": { + "fr": "Office du tourisme", + "en": null, + "es": null, + "it": null + }, + "pictogram": "https://foo.fr/media/upload/desktype-info.svg" + }, + "website": null } ] } \ No newline at end of file diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 36c9a4f7fc..c79a2c793f 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -758,12 +758,12 @@ def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @mock.patch('requests.get') - @mock.patch('requests.head') + @mock.patch('geotrek.common.parsers.AttachmentParserMixin.download_attachment') @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") - def test_create(self, mocked_head, mocked_get): + def test_create(self, mocked_download_attachment, mocked_get): self.mock_time = 0 self.mock_json_order = ['informationdesk.json', ] - self.mock_content_time = 0 + self.mock_download = 0 def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -772,24 +772,21 @@ def mocked_json(): with open(filename, 'r') as f: return json.load(f) - def mocked_requests_get(*args, **kwargs): - response = requests.Response() - response.status_code = 200 - if self.mock_content_time > 0: - response._content = None - else: - response._content = b'' - self.mock_content_time += 1 - response.json = mocked_json - return response + def mocked_download(*args, **kwargs): + if self.mock_download > 0: + return None + self.mock_download += 1 + return b'' # Mock GET - mocked_get.side_effect = mocked_requests_get - + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_download_attachment.side_effect = mocked_download call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekInformationDeskParser', verbosity=0) - self.assertEqual(InformationDesk.objects.count(), 2) + self.assertEqual(InformationDesk.objects.count(), 3) information_desk = InformationDesk.objects.all().first() self.assertEqual(str(information_desk.type), "Relais d'information") self.assertEqual(str(information_desk.name), "Foo") self.assertAlmostEqual(information_desk.geom.x, 573013.9272605104, places=5) self.assertAlmostEqual(information_desk.geom.y, 6276967.321705549, places=5) + self.assertEqual(str(information_desk.photo), '') From 53dde7e786a38b666b1d6f95aaaf5ed14fd9a569 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 2 Aug 2022 14:23:24 +0200 Subject: [PATCH 026/116] Add tests mapping --- geotrek/common/parsers.py | 2 +- geotrek/common/tests/data/organism5.xls | Bin 0 -> 6656 bytes geotrek/common/tests/test_parsers.py | 49 ++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 geotrek/common/tests/data/organism5.xls diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 720f7c90ab..bdd502c7f8 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -363,7 +363,7 @@ def get_mapping(self, src, val, mapping, partial): if mapping is not None: if val and val not in mapping.keys(): values = [str(key) for key in mapping.keys()] - self.add_warning(_("Bad value '{val}' for field {src}. Should be {values}").format(val=str(val), src=src, separator=self.separator, values=values)) + self.add_warning(_("Bad value '{val}' for field {src}. Should be in {values}").format(val=str(val), src=src, separator=self.separator, values=values)) return None if not val: return None diff --git a/geotrek/common/tests/data/organism5.xls b/geotrek/common/tests/data/organism5.xls new file mode 100644 index 0000000000000000000000000000000000000000..9d2244ae8cd1ffb48f7e2010ca7fafe98bbe2de9 GIT binary patch literal 6656 zcmeHLU2IfE6h3#iyWG-3x1~h|!ArHJKiig{sE9!p%8x{Tno^@d5?!{p+tuCfW_ueY zVqA(mkU-S%zzaq(L<17k7=NNB!h=Q=e~1YQ`c!msk)5QQ zXaz1gx^fT8u!$akdp-rwfpUO#6;65+O>I)uB37VV;`mu&$Z8otnw0I>>EdZ7EGBoD zVp`_0tyrJC{`BAU<8GkP`KQn4`L6_~0n>ph09Sxi12w=5pcbeD zW&*bXw*z+ovw+#aoxmJmE^rrM0P}!);BMd^paEzE<^xRtb!~A{=G%~OcghQ#bZqvM zxl&i4T*oeYJ@VPKZ|U-t?>`HRc(3fFvWs89+bRj#MrSY@GX5)CsdNZxw9OMR(q+hL zauD3<@~+~3uD;8Z^+6~g#u6lqNX}~krTaehtykaa_-gX8YV)(=EQDQt4f)KAuQ-Rm zqpg7di%3wSl2;44>s_nMMB`SDTRXR`QV#QOUTY4R$zXIuow>vEXU2erSvp~m&q*o% z`QrFo>G9+X@!9Km^0zMkFxJ%I^c>2rbE~}$>3n{30lvQgzoh`bd|bWDi~rZ?c8=LAqOM;Z4);(_zqZXob00EEQ?q7hfQ{)MclJ_)k$%fcEbZv9(5T%CTEULk_5^!qPYi_phTX{xO8mxJGuaz7 zdV2&}y4lz0)xwx73 z)#JPB>Zyd$^un0vtgDp)`Cuj&kDU8)!?v#FPbz*&Jxo+T(m=zO25l#5PL{Q(7Om<# ztYm8uRLny}(b-tv6N#CF#zym5qdOi84*0vx@K7|EtZ!VnsBuM|G&AhDhFn)LY#NPS zyVL+Lsc$2cS*&Pu`TU%G6Y{&-&J4GmNiT2C$sdw>ys3IIJ^qHpulcoX`)sPoqv%_g z7t|Vxyymmljb*l4^X6FD1<4QhjFoXhv{Z$&&Mro3c5G)yce;xwjQIv0O2}3Tc`W4f zczWH^az*ZvI)QeCCt{a0#CsI5OD>ey$}YK4f;Or|nL4@zr=%f9?uQk>r@Lmg}RN;F3|$>i8eH{`}IFAZ0h&D z^Qtbb^FR2BwzLC)X($_`;@SXHHK=0QJ@V%4(IKt>9`|ZP1vFKjAUFyr4 z79TnZfS08JG6y1G1@7B%N@M`v^AHyBP|NhqDR_RlFLXL&ka2%Y%)Z~oMd$ZX2 s`yqSG>Cz*sFhWS_V5)y0OVek`KbU_9# Date: Tue, 2 Aug 2022 15:43:31 +0200 Subject: [PATCH 027/116] Add help filename import command --- geotrek/common/management/commands/import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/management/commands/import.py b/geotrek/common/management/commands/import.py index c58419b75f..ccac9560eb 100644 --- a/geotrek/common/management/commands/import.py +++ b/geotrek/common/management/commands/import.py @@ -12,7 +12,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('parser', help='Parser class name in var/conf/parsers.py (or dotted syntax in python path)') - parser.add_argument('filename', nargs="?") + parser.add_argument('filename', nargs="?", help='Optional file which will be use to feed database') parser.add_argument('-l', dest='limit', type=int, help='Limit number of lines to import') parser.add_argument('--encoding', '-e', default='utf8') From bfedea35239e5e2bd7cd54ef60bb525de162fe37 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 2 Aug 2022 16:28:11 +0200 Subject: [PATCH 028/116] Add tests parsers --- geotrek/common/parsers.py | 6 +-- geotrek/common/tests/data/attachment.xls | Bin 0 -> 5632 bytes .../tests/data/attachment_no_legend.xls | Bin 0 -> 5632 bytes geotrek/common/tests/test_parsers.py | 40 +++++++++++++++++- 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 geotrek/common/tests/data/attachment.xls create mode 100644 geotrek/common/tests/data/attachment_no_legend.xls diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index bdd502c7f8..1adbd57bba 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -932,10 +932,10 @@ def __init__(self, *args, **kwargs): self.m2m_fields[key] = value self.translated_fields = [field for field in get_translated_fields(self.model)] for category in self.url_categories.keys(): - route = self.url_categories[category] - response = self.request_or_retry(f"{self.url}{route}") - self.field_options.setdefault(category, {}) if self.categories_keys_api_v2.get(category): + route = self.url_categories[category] + response = self.request_or_retry(f"{self.url}{route}") + self.field_options.setdefault(category, {}) self.field_options[category]["mapping"] = {} results = response.json()['results'] for result in results: diff --git a/geotrek/common/tests/data/attachment.xls b/geotrek/common/tests/data/attachment.xls new file mode 100644 index 0000000000000000000000000000000000000000..71083411e8562558a03a9faaa712e4f496419506 GIT binary patch literal 5632 zcmeHLU2IfE6h3#`Ew{ALZK0?jc&V`bblX}Xh!|w~RiZ#sF&aanOMAOr+3q%FtB{~B zMcyFM@W2a+k`N7Q1Y@E`O)M`)6Mu*a32I`BPoh3724k8~*6%y_ZhP76rdzBgn4Zly zKWEO&oS8Xu&fGo!ZSBy>_vT)bJbFONL|pawt)kXoP)m<7xR>VY}H zZNTlo9l%^*9&jfxA6Nj~1yGNLZqD)|loz}8C2l^pvZ>O)u>+MRPO-u-pUn7*E?<5B zC0NAS@^fkzKZCbRdTARQBk>OFzoHGQL&TtM4#G&+qGZSkaA(RH<$kK^TD3k1g~eKp zj4zU_Mp$(}s%e9!Gm#qdq4xPfIjykEFCd?N=@s`9_>J}Oe=&(jT=H5WH`%S#QfS@< z<<)YE(f1)cVfRJiLwc)D$)8yZI^<~3BA;q2|7vM`?qwnQVtkHTNdEuJKZQNDxIC9~ z``qa4LnfcUtO!3;gulE9zp^0T4@*#eMjzdS)-fM$Vp_7y1L-o&Ytt2)*QLufpOv1b z`Ruem`;6Y3!|>HVq9fn$TkQT=JZ`rh!JS@+-#NnsM0OrEntKz`kQKM1cA_(6B_cg`7SYI*eb9-~sKcxW zP;iOShvZ|(a$H;B2G>@NrF3G6bOyD$tN@0)cKXV9+jh6Dds_KvaB{Sm+l_S}M9aDI z<=UfB(*c#8EsL;)65JSU*ct1w_gdTS=dJc+Pb3j)x1;^>NMA#+_1@t6dRZpHapl?~ zQQHc(9nuM2)-(uZRw!Lwx5QIkit?eRivvv;GwYUl$|t1(F}oAX<8MGhM##wZ&*g?Z zjbjhXq~;zE*h)jMK(6rl0OQ=jgFZD6fzir+36$Wb=8uZp8xXK z{lg!BQ+s4U-jYX7SMjd!^}|&YD(jh_rIsL8~}LuY^UOU z--ZG1)}sL5xlaQ0E}a11yS)IFygcH(U0GfQgaPUASu+A7(K73xFmOEfd3e=<^Yn*u zX61%oo_ph>8ZN3gP%-^EhwsAI7fv+K3miKR{X@g=ZUWDr$J+%Sy)*|Wcmvp%x)$T3 zd6Xve`%mZ3v!T(6JO%oH*CEIgxD$P{M|SIX=pK($X~jCMmg~jkRvRv@m&i|eVv!8l zh?#qE#ygUcz;NxbhRL~)fF@_Z(QtBx)qd}j`^LLowT4(9;eyfrE z@|g5v{XOtV{Py1pFVv-HLJqxC5}e|+)SaK6wbb?B4cT|xA^oxmGjym9w*CV#F@09w Y#QG74v9igdiSEDm`+G99_$TxK4Ud{x@c;k- literal 0 HcmV?d00001 diff --git a/geotrek/common/tests/data/attachment_no_legend.xls b/geotrek/common/tests/data/attachment_no_legend.xls new file mode 100644 index 0000000000000000000000000000000000000000..c0f7f9e62072126b6f7922eb42bca5d22efb5ab3 GIT binary patch literal 5632 zcmeHLU2IfE6h3#m+ur^_w}qmD;HA>#r==Azf*53>fD#32t)N3x{G#T zEUhcg(2GO#0X*{(fV|Rlc^55dM%4m#pjBe{#l@1X(u=ZR4&ub~PxD|g?&Gl`- z$lp$8X8wsCxZPf{-M#bX~f>G+j=tY=-(bcA(V6DHi+W%UR#k<*V<% z42$^4dtUA0SMYXAA8n&K6z#D7tF&2l2pP1^AsFcfP==fWceZ?}+^;m>pw}i_XBF)7Tgazge=~gve8wjDzlek+Dp{?No9@;MnQY!g z-U=CH^nJqawfjTS5xrG|@@L9|4w*D$kx#Xie?31w_i{4%Tzrl?nf(8kKZrfGxV&`A z>~p=d56Nu)@*Mnz9Q+kI_@$HbeXs=8XZ+DUWS#KfCMG1s(w`{OvMN!mWp%=<<(x!` zmU9!n)XRErj>1=8Ku5mcx7qt5(Wt%Z81D39{LULDAkuc+kl*D+diG?iF0?pF56-Jk z3!$~Su|~?1^!#6AOj8aUh!o}!Zl0JyS#*uD0kOhF%rFrP%)byWZpdnlGmJIB&e#sT zHe0%5@fe~+cRb#=zNsmWT4OvE4>iRj@knD|Z#aecW6A+&!T8f*)&L;ViSdTX!yx52 zGyg5ltOCk+Vo|*QRk{#AhP!d*+D|*$TQ$QMK1$=WaXuSm6wtmm%HfGQi~|viRJM(APq)?k?Ef)4S5b@oARpep~#@0 z@msXcUgwrf)N$5EqTH#Ahq2x{epEtQo`K3um8&q7$BW2o;mdo#% z$34~ZseZ@?T~Z4!xN?eg$%}AcKY0SN42D;!{7aOaP*5Gs*VlF z`|{M8a^4QUclgff#v3}0c)4Dh@Qdr)Iwp;vLJIOCibhEFN*H}cvc`M(4#(jYC^Q*( zg%zPA*XcL862x20WgFjYJ0czZvAwabxV7~_haI&ZUAwX^)NaFp=u=^Hq^rvbyH1~i zbDXU+X~8|64mWW+#w0_~W=4*_{m1C8ZvO{wnNq*>lb>ljhX9%a+o?F;w?Tk=_cXwF z?h62YO(($jZXZA;uZ%fwSJG>NAdqN3`^*?jhIA$cL7a?y6MW>*F#Vx)y?5g3{&z2_ z8KZhj%B4T&@Larb@l@ji|A~{(zhU&;7Vvyoyxri@OLK67_kn$>Yc4*T$J2Cv|LOc$ zHZ(qwi=h8cy9aq5H>6+oNV|TA?r}-wM^ZQyJ!WW_I$arJdGV6#Q0W^2xKZ59{xN zf8w|QcKD<&Jri>1t772fPRr8s)6eF+{s$p@Ha( Date: Wed, 3 Aug 2022 10:14:41 +0200 Subject: [PATCH 029/116] Improve helper command import filename --- geotrek/common/management/commands/import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/management/commands/import.py b/geotrek/common/management/commands/import.py index ccac9560eb..e0d0eadf6f 100644 --- a/geotrek/common/management/commands/import.py +++ b/geotrek/common/management/commands/import.py @@ -12,7 +12,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('parser', help='Parser class name in var/conf/parsers.py (or dotted syntax in python path)') - parser.add_argument('filename', nargs="?", help='Optional file which will be use to feed database') + parser.add_argument('filename', nargs="?", help='Optional file used to feed database') parser.add_argument('-l', dest='limit', type=int, help='Limit number of lines to import') parser.add_argument('--encoding', '-e', default='utf8') From 2c251f078d88cde4896dd44807506a4742ddea94 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Wed, 3 Aug 2022 14:41:46 +0200 Subject: [PATCH 030/116] Add Docstring, comment on GeotrekParser --- geotrek/common/parsers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 1adbd57bba..192fe14dfa 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -896,6 +896,12 @@ def normalize_field_name(self, name): class GeotrekParser(AttachmentParserMixin, Parser): + """ + url_categories: url of the categories in api v2 (example: 'category': '/api/v2/touristiccontent_category/') + replace_fields: Replace fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) + m2m_replace_fields: Replace m2m fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) + categories_keys_api_v2: Key in the route of the category (example: /api/v2/touristiccontent_category/) corresponding to the model field + """ model = None next_url = '' url = None @@ -925,12 +931,14 @@ def __init__(self, *args, **kwargs): f.name: f.name for f in self.model._meta.many_to_many } + # Replace automatics fields and m2m_fields generated above for key, value in self.replace_fields.items(): self.fields[key] = value for key, value in self.m2m_replace_fields.items(): self.m2m_fields[key] = value self.translated_fields = [field for field in get_translated_fields(self.model)] + # Generate a mapping dictionnary between id and the correspondant label for category in self.url_categories.keys(): if self.categories_keys_api_v2.get(category): route = self.url_categories[category] @@ -938,6 +946,7 @@ def __init__(self, *args, **kwargs): self.field_options.setdefault(category, {}) self.field_options[category]["mapping"] = {} results = response.json()['results'] + # for element in category url map the id with its label for result in results: id_result = result['id'] label = result[self.categories_keys_api_v2[category]] From f9528ffe3cad3f0e7a1346323e20cad97e06666d Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Wed, 3 Aug 2022 16:06:22 +0200 Subject: [PATCH 031/116] Add label on Trekking parser api v2 --- geotrek/trekking/parsers.py | 7 ++- .../data/geotrek_parser_v2/trek_label.json | 58 +++++++++++++++++++ geotrek/trekking/tests/test_parsers.py | 5 +- 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 9ae5920194..3752f22862 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -82,7 +82,8 @@ class GeotrekTrekParser(GeotrekParser): "themes": "/api/v2/theme/", "practice": "/api/v2/trek_practice/", "accessibilities": "/api/v2/trek_accessibility/", - "networks": "/api/v2/trek_network/" + "networks": "/api/v2/trek_network/", + 'labels': '/api/v2/label/' } categories_keys_api_v2 = { 'difficulty': 'label', @@ -90,7 +91,8 @@ class GeotrekTrekParser(GeotrekParser): 'themes': 'label', 'practice': 'name', 'accessibilities': 'name', - 'networks': 'label' + 'networks': 'label', + 'labels': 'name' } natural_keys = { 'difficulty': 'difficulty', @@ -99,6 +101,7 @@ class GeotrekTrekParser(GeotrekParser): 'practice': 'name', 'accessibilities': 'name', 'networks': 'network', + 'labels': 'name' } def next_row(self): diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json new file mode 100644 index 0000000000..6134e64ec6 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json @@ -0,0 +1,58 @@ +{ + "count": 3, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "advice": { + "fr": "

La divagation des chiens est autorisée sur le sentier.

\r\n

\"Chient

", + "en": "The national park is an unrestricted natural area but subjected to regulations which must be known by all visitors.", + "es": "El parque nacional es un área natural sin restricciones pero sometido a regulaciones que deben ser conocidas por todos los visitantes.", + "it": "Il Parco Nazionale è un territorio naturale, aperto a tutti, ma soggetto ad un regolamento che è utile conoscere per preparare il vostro soggiorno." + }, + "filter": true, + "name": { + "fr": "Chien autorisé", + "en": "Is in the midst of the park", + "es": "En coeur de parc", + "it": "Nel cuore del parco" + }, + "pictogram": "https://demo-admin.geotrek.fr/media/upload/dog.png" + }, + { + "id": 2, + "advice": { + "fr": "Le Parc national est un territoire naturel, ouvert à tous, mais soumis à une réglementation qu’il est utile de connaître pour préparer son séjour", + "en": null, + "es": null, + "it": null + }, + "filter": true, + "name": { + "fr": "En coeur de parc", + "en": null, + "es": null, + "it": null + }, + "pictogram": null + }, + { + "id": 3, + "advice": { + "fr": "Cette randonnée se déroule sur un territoire où les patous sont présents.", + "en": null, + "es": null, + "it": null + }, + "filter": true, + "name": { + "fr": "Information Patou", + "en": null, + "es": null, + "it": null + }, + "pictogram": null + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index d430f9d284..501f2292b8 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -142,6 +142,7 @@ class TestGeotrekTrekParser(GeotrekTrekParser): 'accessibilities': {'create': True}, 'networks': {'create': True}, 'geom': {'required': True}, + 'labels': {'create': True}, } @@ -175,7 +176,7 @@ def setUpTestData(cls): def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', - 'trek_accessibility.json', 'trek_network.json', 'trek.json', 'trek_children.json'] + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', 'trek_children.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -199,6 +200,8 @@ def mocked_json(): self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) self.assertAlmostEqual(trek.geom[0][1], 6190964.893167565, places=5) self.assertEqual(trek.children.first().name, "Foo") + self.assertEqual(trek.labels.count(), 3) + self.assertEqual(trek.labels.first().name, "Chien autorisé") @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') From 4cf0842aeb86ec028cdf67208cc94b41070c3f2a Mon Sep 17 00:00:00 2001 From: Emmanuelle Helly Date: Thu, 4 Aug 2022 10:49:45 +0200 Subject: [PATCH 032/116] Add light documentation --- docs/install/import.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/install/import.rst b/docs/install/import.rst index 5238dcb4e4..8888fb45a3 100644 --- a/docs/install/import.rst +++ b/docs/install/import.rst @@ -195,6 +195,37 @@ Start import from Geotrek-admin UI Open the top right menu and clic on ``imports``. +Import data from a remote Geotrek instance +========================================== + +Importing from a Geotrek instance works the same way as from SIT. + +For example, to import treks from another instance, +edit ``/opt/geotrek-admin/var/conf/parsers.py`` file with the following content: + +:: + + class DemoGeotrekTrekParser(BaseGeotrekTrekParser): + url = "https://remote-geotrek-admin.net" # replace url with remote instance url + delete = False + field_options = { + 'difficulty': {'create': True, }, + 'route': {'create': True, }, + 'themes': {'create': True}, + 'practice': {'create': True}, + 'accessibilities': {'create': True}, + 'networks': {'create': True}, + 'geom': {'required': True}, + 'labels': {'create': True}, + } + +Then run in command line + +:: + + sudo geotrek import DemoGeotrekTrekParser + + Import data from a file ======================= From ffaef8a8e10a46c5dd47616f59635491215f9fff Mon Sep 17 00:00:00 2001 From: Emmanuelle Helly Date: Thu, 4 Aug 2022 10:55:38 +0200 Subject: [PATCH 033/116] Add warning about dynamic segmentation --- docs/install/import.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/install/import.rst b/docs/install/import.rst index 8888fb45a3..287538b2e8 100644 --- a/docs/install/import.rst +++ b/docs/install/import.rst @@ -199,6 +199,12 @@ Import data from a remote Geotrek instance ========================================== Importing from a Geotrek instance works the same way as from SIT. +A usecase for this is to aggregate data from several Geotrek-admin instance. + +.. danger:: + Importing data from a remote Geotrek instance does not work with dynamic segmentation, your instance where you import data + must have dynamic segmentation disabled. + For example, to import treks from another instance, edit ``/opt/geotrek-admin/var/conf/parsers.py`` file with the following content: @@ -225,6 +231,8 @@ Then run in command line sudo geotrek import DemoGeotrekTrekParser +Treks are now imported into your own instance. + Import data from a file ======================= From 5b2b425433f9a18f836a1b3e7c0e8f47a113fa76 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 14:03:27 +0200 Subject: [PATCH 034/116] Add test different eid, fix multiple same parser --- geotrek/common/parsers.py | 14 ++ geotrek/trekking/parsers.py | 7 +- .../tests/data/geotrek_parser_v2/trek_2.json | 217 ++++++++++++++++++ .../data/geotrek_parser_v2/trek_label.json | 2 +- geotrek/trekking/tests/test_parsers.py | 46 ++++ 5 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 192fe14dfa..931f6c8052 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -901,6 +901,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): replace_fields: Replace fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) m2m_replace_fields: Replace m2m fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) categories_keys_api_v2: Key in the route of the category (example: /api/v2/touristiccontent_category/) corresponding to the model field + eid_prefix: Prefix of your eid which allow to differentiate multiple GeotrekParser """ model = None next_url = '' @@ -920,6 +921,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): 'geom': {'required': True}, } bbox = None + eid_prefix = '' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -960,6 +962,18 @@ def __init__(self, *args, **kwargs): raise ImproperlyConfigured(f"{category} is not configured in categories_keys_api_v2") self.creator, created = get_user_model().objects.get_or_create(username='import', defaults={'is_active': False}) + def start(self): + super().start() + kwargs = self.get_to_delete_kwargs() + kwargs['eid__startswith'] = self.eid_prefix + if kwargs is None: + self.to_delete = set() + else: + self.to_delete = set(self.model.objects.filter(**kwargs).values_list('pk', flat=True)) + + def filter_eid(self, src, val): + return f'{self.eid_prefix}{val}' + def filter_attachments(self, src, val): return [(subval.get('url'), subval.get('legend'), subval.get('author')) for subval in val] diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 3752f22862..24272c17ab 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -132,12 +132,15 @@ def end(self): for key, value in final_children.items(): if value['children']: - trek_parent_instance = Trek.objects.get(eid=value['uuid']) + trek_parent_instance = Trek.objects.filter(eid=value['uuid']) + if not trek_parent_instance: + return order = 0 for child in value['children']: trek_child_uuid = final_children.get(child)['uuid'] trek_child_instance = Trek.objects.get(eid=trek_child_uuid) - OrderedTrekChild.objects.create(parent=trek_parent_instance, child=trek_child_instance, order=order) + OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, + order=order) order += 1 diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json new file mode 100644 index 0000000000..82e8aa2989 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json @@ -0,0 +1,217 @@ +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "id": 8702, + "access": { + "fr": "Accès", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [], + "accessibility_advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_covering": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_exposure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_slope": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_width": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://foo.fr/api/v2/trek/8702/profile/", + "ambiance": { + "fr": "Ambiance", + "en": "", + "es": "", + "it": "" + }, + "arrival": { + "fr": "Étangs de Picot", + "en": "", + "es": "", + "it": "" + }, + "ascent": 797, + "attachments": [], + "attachments_accessibility": [], + "children": [], + "cities": [ + "09030" + ], + "create_datetime": "2019-04-01T13:04:06.795861Z", + "departure": { + "fr": "Barrage de Soulcem", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09030", + "departure_geom": [ + 1.4526560730280935, + 42.677959776888834 + ], + "descent": -65, + "description": { + "fr": "
Description Lorem ipsum sit dolor amet Description Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor amet

\r\n
    \r\n
  1. \r\n
    2tape numéro 1 sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDesc
    \r\n
    2. \r\n
  2. \r\n
    Prendre à droite ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lor
    \r\n
    3. \r\n
  3. Terminer tout droit
    4. \r\n
\r\n
Description Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor amet
", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "Chapeau", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 3, + "disabled_infrastructure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "duration": 5.0, + "elevation_area_url": "https://foo.fr/api/v2/trek/8702/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/8702/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 1.4526561, + 42.6779598, + 1552.0 + ], + [ + 1.4526694, + 42.6779712, + 1554.0 + ] + ] + }, + "gpx": "https://foo.fr/api/fr/treks/8702/etangs-du-picot.gpx", + "information_desks": [], + "kml": "https://foo.fr/api/fr/treks/8702/etangs-du-picot.kml", + "labels": [], + "length_2d": 3347.5, + "length_3d": 3536.2, + "max_elevation": 2298, + "min_elevation": 1537, + "name": { + "fr": "Étangs du Picot", + "en": "", + "es": "", + "it": "" + }, + "networks": [], + "next": {}, + "parents": [], + "parking_location": null, + "pdf": { + "fr": "https://foo.fr/api/fr/treks/8702/etangs-du-picot.pdf", + "en": "https://foo.fr/api/en/treks/8702/etangs-du-picot.pdf", + "es": "https://foo.fr/api/es/treks/8702/etangs-du-picot.pdf", + "it": "https://foo.fr/api/it/treks/8702/etangs-du-picot.pdf" + }, + "points_reference": { + "type": "MultiPoint", + "coordinates": [ + [ + 1.451697349548341, + 42.6817728861541 + ], + [ + 1.456890106201172, + 42.68590558513195 + ], + [ + 1.465902328491211, + 42.68448598671003 + ] + ] + }, + "portal": [], + "practice": 4, + "ratings": [], + "ratings_description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "previous": {}, + "public_transport": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 2, + "second_external_id": null, + "source": [], + "structure": 3, + "themes": [], + "update_datetime": "2022-04-11T14:52:42.637165Z", + "url": "https://foo.fr/api/v2/trek/8702/", + "uuid": "58ed4fc1-645d-4bf6-b956-71f0a01a5eec", + "web_links": [] + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json index 6134e64ec6..cb5aecc593 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json @@ -18,7 +18,7 @@ "es": "En coeur de parc", "it": "Nel cuore del parco" }, - "pictogram": "https://demo-admin.geotrek.fr/media/upload/dog.png" + "pictogram": "https://foo.fr/media/upload/dog.png" }, { "id": 2, diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 501f2292b8..743592df54 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -146,6 +146,15 @@ class TestGeotrekTrekParser(GeotrekTrekParser): } +class TestGeotrek2TrekParser(GeotrekTrekParser): + url = "https://test.fr" + + field_options = { + 'geom': {'required': True}, + } + eid_prefix = 'geotrek2' + + class TestGeotrekPOIParser(GeotrekPOIParser): url = "https://test.fr" @@ -203,6 +212,43 @@ def mocked_json(): self.assertEqual(trek.labels.count(), 3) self.assertEqual(trek.labels.first().name, "Chien autorisé") + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_create_multiple(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', + 'trek_children.json', 'trek_difficulty.json', 'trek_route.json', 'trek_theme.json', + 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', + 'trek_2.json', 'trek_children.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 5) + trek = Trek.objects.all().first() + self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(str(trek.difficulty), 'Très facile') + self.assertEqual(str(trek.practice), 'Cheval') + self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) + self.assertAlmostEqual(trek.geom[0][1], 6190964.893167565, places=5) + self.assertEqual(trek.children.first().name, "Foo") + self.assertEqual(trek.labels.count(), 3) + self.assertEqual(trek.labels.first().name, "Chien autorisé") + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrek2TrekParser', verbosity=2) + self.assertEqual(Trek.objects.count(), 6) + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class POIGeotrekParserTests(TestCase): From 54b98963c13c7ed7c45f5cf2466c2fbc0dbc4fb8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 14:33:07 +0200 Subject: [PATCH 035/116] Use id as eid for parser --- geotrek/common/parsers.py | 2 +- geotrek/infrastructure/parsers.py | 2 +- geotrek/signage/parsers.py | 2 +- geotrek/tourism/parsers.py | 4 ++-- geotrek/trekking/parsers.py | 19 +++++++++---------- geotrek/trekking/tests/test_parsers.py | 2 +- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 931f6c8052..3adc906e68 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -908,7 +908,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): url = None separator = None delete = True - eid = 'uuid' + eid = 'eid' constant_fields = {} url_categories = {} replace_fields = {} diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py index 7cb727a80c..490a07823b 100644 --- a/geotrek/infrastructure/parsers.py +++ b/geotrek/infrastructure/parsers.py @@ -10,7 +10,7 @@ class GeotrekInfrastructureParser(GeotrekParser): "deleted": False } replace_fields = { - "eid": "uuid", + "eid": "id", "geom": "geometry" } url_categories = { diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py index 86ec5a3cc9..df07d82a73 100644 --- a/geotrek/signage/parsers.py +++ b/geotrek/signage/parsers.py @@ -10,7 +10,7 @@ class GeotrekSignageParser(GeotrekParser): "deleted": False } replace_fields = { - "eid": "uuid", + "eid": "id", "geom": "geometry" } url_categories = { diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 3110663d80..1761150d64 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -899,7 +899,7 @@ class GeotrekTouristicContentParser(GeotrekParser): } replace_fields = { - "eid": "uuid", + "eid": "id", "geom": "geometry", } @@ -974,7 +974,7 @@ class GeotrekTouristicEventParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "uuid", + "eid": "id", "geom": "geometry" } url_categories = { diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 24272c17ab..9411fb3889 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -72,7 +72,7 @@ class GeotrekTrekParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "uuid", + "eid": "id", "eid2": "second_external_id", "geom": "geometry" } @@ -122,23 +122,22 @@ def end(self): self.next_url = f"{self.url}/api/v2/trek" params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'fields': ['children', 'id', 'uuid'] + 'fields': ['children', 'id'] } response = self.request_or_retry(f"{self.next_url}", params=params) results = response.json()['results'] final_children = {} for result in results: - final_children[result['id']] = {'uuid': result['uuid'], 'children': result['children']} + final_children[result['id']] = result['children'] for key, value in final_children.items(): - if value['children']: - trek_parent_instance = Trek.objects.filter(eid=value['uuid']) + if value: + trek_parent_instance = Trek.objects.filter(eid=f"{self.eid_prefix}{key}") if not trek_parent_instance: return order = 0 - for child in value['children']: - trek_child_uuid = final_children.get(child)['uuid'] - trek_child_instance = Trek.objects.get(eid=trek_child_uuid) + for child in value: + trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, order=order) order += 1 @@ -151,7 +150,7 @@ class GeotrekServiceParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "uuid", + "eid": "id", "geom": "geometry" } url_categories = { @@ -177,7 +176,7 @@ class GeotrekPOIParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "uuid", + "eid": "id", "geom": "geometry" } url_categories = { diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 743592df54..17f7613bc2 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -246,7 +246,7 @@ def mocked_json(): self.assertEqual(trek.children.first().name, "Foo") self.assertEqual(trek.labels.count(), 3) self.assertEqual(trek.labels.first().name, "Chien autorisé") - call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrek2TrekParser', verbosity=2) + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrek2TrekParser', verbosity=0) self.assertEqual(Trek.objects.count(), 6) From 3f15d6e7d26f9c0bb338641ecb8903ace419447a Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 15:14:29 +0200 Subject: [PATCH 036/116] Add exception message in children generation --- geotrek/common/parsers.py | 3 +- geotrek/trekking/parsers.py | 47 ++++++++++--------- .../data/geotrek_parser_v2/trek_children.json | 15 ++---- geotrek/trekking/tests/test_parsers.py | 26 ++++++++++ 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 3adc906e68..fbf52e6d64 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -928,7 +928,8 @@ def __init__(self, *args, **kwargs): self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID self.bbox.transform(4326) # WGS84 - self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if + not isinstance(f, TranslationField) and not f.name == 'id') self.m2m_fields = { f.name: f.name for f in self.model._meta.many_to_many diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 9411fb3889..61b89f9bcf 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -3,7 +3,7 @@ from django.contrib.gis.geos import Point, GEOSGeometry from django.utils.translation import gettext as _ -from geotrek.common.parsers import ShapeParser, AttachmentParserMixin, GeotrekParser +from geotrek.common.parsers import ShapeParser, AttachmentParserMixin, GeotrekParser, RowImportError from geotrek.trekking.models import OrderedTrekChild, POI, Service, Trek @@ -120,27 +120,30 @@ def filter_points_reference(self, src, val): def end(self): super().end() self.next_url = f"{self.url}/api/v2/trek" - params = { - 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'fields': ['children', 'id'] - } - response = self.request_or_retry(f"{self.next_url}", params=params) - results = response.json()['results'] - final_children = {} - for result in results: - final_children[result['id']] = result['children'] - - for key, value in final_children.items(): - if value: - trek_parent_instance = Trek.objects.filter(eid=f"{self.eid_prefix}{key}") - if not trek_parent_instance: - return - order = 0 - for child in value: - trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") - OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, - order=order) - order += 1 + try: + params = { + 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), + 'fields': ['children', 'id'] + } + response = self.request_or_retry(f"{self.next_url}", params=params) + results = response.json()['results'] + final_children = {} + for result in results: + final_children[result['id']] = result['children'] + + for key, value in final_children.items(): + if value: + trek_parent_instance = Trek.objects.filter(eid=f"{self.eid_prefix}{key}") + if not trek_parent_instance: + return + order = 0 + for child in value: + trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") + OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, + order=order) + order += 1 + except Exception as e: + self.add_warning(_(f"An error occured in children generation : {e}")) class GeotrekServiceParser(GeotrekParser): diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json index 1bb157408d..3cb7aafe61 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json @@ -7,28 +7,23 @@ "id": 2, "children": [ 10439 - ], - "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1" + ] }, { "id": 10443, - "children": [], - "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12" + "children": [] }, { "id": 2849, - "children": [], - "uuid": "6761143f-9244-41d0-b1af-21114408f769" + "children": [] }, { "id": 10439, - "children": [], - "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" + "children": [] }, { "id": 8700, - "children": [], - "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1" + "children": [] } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 17f7613bc2..ed899302c4 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -1,3 +1,4 @@ +from io import StringIO from unittest import mock import json import os @@ -249,6 +250,31 @@ def mocked_json(): call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrek2TrekParser', verbosity=0) self.assertEqual(Trek.objects.count(), 6) + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_wrong_children_error(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', + 'trek_wrong_children.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + output = StringIO() + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, + stdout=output) + self.assertIn("An error occured in children generation", output.getvalue()) + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class POIGeotrekParserTests(TestCase): From 3a8e2b08318b76087a5b09603db25d406d14ceb9 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 15:39:02 +0200 Subject: [PATCH 037/116] Remove rowimporterror --- geotrek/trekking/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 61b89f9bcf..974cb9ab45 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -3,7 +3,7 @@ from django.contrib.gis.geos import Point, GEOSGeometry from django.utils.translation import gettext as _ -from geotrek.common.parsers import ShapeParser, AttachmentParserMixin, GeotrekParser, RowImportError +from geotrek.common.parsers import ShapeParser, AttachmentParserMixin, GeotrekParser from geotrek.trekking.models import OrderedTrekChild, POI, Service, Trek From 53096b6a6be57194bad9e5c542ca658fb74f72cb Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 15:40:47 +0200 Subject: [PATCH 038/116] Add wrong children test --- .../trek_wrong_children.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json new file mode 100644 index 0000000000..6fa8b5887c --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json @@ -0,0 +1,30 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 2, + "children": [ + 10439, + 10888 + ] + }, + { + "id": 10443, + "children": [] + }, + { + "id": 2849, + "children": [] + }, + { + "id": 10439, + "children": [] + }, + { + "id": 8700, + "children": [] + } + ] +} \ No newline at end of file From 16431feb7a7a89d1241761ed72048994c002e480 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 15:53:15 +0200 Subject: [PATCH 039/116] Add warnings, addportal_filter, url on general parser --- geotrek/common/parsers.py | 11 ++++++++--- geotrek/trekking/parsers.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index fbf52e6d64..3a4a94dd65 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -902,6 +902,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): m2m_replace_fields: Replace m2m fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) categories_keys_api_v2: Key in the route of the category (example: /api/v2/touristiccontent_category/) corresponding to the model field eid_prefix: Prefix of your eid which allow to differentiate multiple GeotrekParser + portals_filter: Portals which will be use for filter in api v2 (default: No portal filter) """ model = None next_url = '' @@ -922,14 +923,16 @@ class GeotrekParser(AttachmentParserMixin, Parser): } bbox = None eid_prefix = '' + portals_filter = None - def __init__(self, *args, **kwargs): + def __init__(self, url=None, portals_filter=None, *args, **kwargs): super().__init__(*args, **kwargs) self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID self.bbox.transform(4326) # WGS84 - self.fields = dict((f.name, f.name) for f in self.model._meta.fields if - not isinstance(f, TranslationField) and not f.name == 'id') + self.portals_filter = portals_filter + self.url = url if url else self.url + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') self.m2m_fields = { f.name: f.name for f in self.model._meta.many_to_many @@ -1001,8 +1004,10 @@ def filter_geom(self, src, val): def next_row(self): while self.next_url: + portals = self.portals_filter params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), + 'portals': ','.join(portals) if portals else '' } response = self.request_or_retry(self.next_url, params=params) self.root = response.json() diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 974cb9ab45..1b4cb6d1a9 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -123,7 +123,7 @@ def end(self): try: params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'fields': ['children', 'id'] + 'fields': 'children,id' } response = self.request_or_retry(f"{self.next_url}", params=params) results = response.json()['results'] @@ -138,8 +138,13 @@ def end(self): return order = 0 for child in value: - trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") - OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, + try: + trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") + except Trek.DoesNotExist: + self.add_warning(_(f"One trek has not be generated for {trek_parent_instance[0].pk}")) + continue + OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], + child=trek_child_instance, order=order) order += 1 except Exception as e: From ef7f2a14a2e3f8a4a96fade4a14b9c1548365a31 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 4 Aug 2022 15:55:41 +0200 Subject: [PATCH 040/116] Improve warning use name for generation of children --- geotrek/trekking/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 1b4cb6d1a9..e411bdfd36 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -141,7 +141,7 @@ def end(self): try: trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") except Trek.DoesNotExist: - self.add_warning(_(f"One trek has not be generated for {trek_parent_instance[0].pk}")) + self.add_warning(_(f"One trek has not be generated for {trek_parent_instance[0].name}")) continue OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, From a9ba8b169ad7c71a45a07775592195ad684e7f34 Mon Sep 17 00:00:00 2001 From: Emmanuelle Helly Date: Thu, 4 Aug 2022 18:30:03 +0200 Subject: [PATCH 041/116] Add some docstring --- geotrek/common/parsers.py | 7 ++++++- geotrek/infrastructure/parsers.py | 2 ++ geotrek/signage/parsers.py | 2 ++ geotrek/tourism/parsers.py | 6 ++++++ geotrek/trekking/parsers.py | 7 +++++++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 3a4a94dd65..6a3401bccc 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -944,7 +944,7 @@ def __init__(self, url=None, portals_filter=None, *args, **kwargs): for key, value in self.m2m_replace_fields.items(): self.m2m_fields[key] = value self.translated_fields = [field for field in get_translated_fields(self.model)] - # Generate a mapping dictionnary between id and the correspondant label + # Generate a mapping dictionnary between id and the related label for category in self.url_categories.keys(): if self.categories_keys_api_v2.get(category): route = self.url_categories[category] @@ -1003,6 +1003,10 @@ def filter_geom(self, src, val): return geom def next_row(self): + """Returns next row. + Geotrek API is paginated, run until "next" is empty + :returns row + """ while self.next_url: portals = self.portals_filter params = { @@ -1015,4 +1019,5 @@ def next_row(self): for row in self.items: yield row + self.next_url = self.root['next'] diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py index 490a07823b..9eee9e7a91 100644 --- a/geotrek/infrastructure/parsers.py +++ b/geotrek/infrastructure/parsers.py @@ -3,6 +3,8 @@ class GeotrekInfrastructureParser(GeotrekParser): + """Geotrek parser for Infrastructure""" + url = None model = Infrastructure constant_fields = { diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py index df07d82a73..5720c80f67 100644 --- a/geotrek/signage/parsers.py +++ b/geotrek/signage/parsers.py @@ -3,6 +3,8 @@ class GeotrekSignageParser(GeotrekParser): + """Geotrek parser for Signage""" + url = None model = Signage constant_fields = { diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 1761150d64..a2796f48a6 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -891,6 +891,8 @@ class TouristicEventTourInSoftParserV3(TouristicEventTourInSoftParser): class GeotrekTouristicContentParser(GeotrekParser): + """Geotrek parser for TouristicContent""" + url = None model = TouristicContent constant_fields = { @@ -932,6 +934,7 @@ class GeotrekTouristicContentParser(GeotrekParser): } def __init__(self, *args, **kwargs): + """Initialize parser with mapping for type1 and type2""" super().__init__(*args, **kwargs) response = self.request_or_retry(f"{self.url}/api/v2/touristiccontent_category/", ) self.field_options.setdefault("type1", {}) @@ -967,6 +970,8 @@ def filter_type2(self, src, val): class GeotrekTouristicEventParser(GeotrekParser): + """Geotrek parser for TouristicEvent""" + url = None model = TouristicEvent constant_fields = { @@ -993,6 +998,7 @@ def next_row(self): class GeotrekInformationDeskParser(GeotrekParser): + """Geotrek parser for InformationDesk""" url = None model = InformationDesk constant_fields = {} diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index e411bdfd36..22fa044bd5 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -65,6 +65,8 @@ def filter_geom(self, src, val): class GeotrekTrekParser(GeotrekParser): + """Geotrek parser for Trek""" + url = None model = Trek constant_fields = { @@ -118,6 +120,7 @@ def filter_points_reference(self, src, val): return geom.transform(settings.SRID, clone=True) def end(self): + """Add children after all treks imported are created in database.""" super().end() self.next_url = f"{self.url}/api/v2/trek" try: @@ -152,6 +155,8 @@ def end(self): class GeotrekServiceParser(GeotrekParser): + """Geotrek parser for Service""" + url = None model = Service constant_fields = { @@ -177,6 +182,8 @@ def next_row(self): class GeotrekPOIParser(GeotrekParser): + """Geotrek parser for GeotrekPOI""" + url = None model = POI constant_fields = { From 90cfda68e92e52ba493f71449908ff408eb77b2f Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 5 Aug 2022 14:38:06 +0200 Subject: [PATCH 042/116] Add aggregator parser --- geotrek/common/parsers.py | 93 ++++++++++++++++--- .../common/parser_report_aggregator.txt | 8 ++ geotrek/infrastructure/parsers.py | 4 +- geotrek/signage/parsers.py | 6 +- geotrek/tourism/parsers.py | 8 +- geotrek/trekking/parsers.py | 18 ++-- 6 files changed, 107 insertions(+), 30 deletions(-) create mode 100644 geotrek/common/templates/common/parser_report_aggregator.txt diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 6a3401bccc..2e30b62680 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1,4 +1,5 @@ from io import BytesIO +import importlib import json import os import re @@ -895,6 +896,64 @@ def normalize_field_name(self, name): return name +class GeotrekAggregatorParser: + filename = None + url = None + + mapping_model_parser = { + "Trek": ("geotrek.trekking.parsers", "GeotrekTrekParser"), + "POI": ("geotrek.trekking.parsers", "GeotrekPOIParser"), + "Service": ("geotrek.trekking.parsers", "GeotrekServiceParser"), + "InformationDesk": ("geotrek.tourism.parsers", "GeotrekInformationDeskParser"), + "TouristicContent": ("geotrek.tourism.parsers", "GeotrekTouristicContentParser"), + "TouristicEvent": ("geotrek.tourism.parsers", "GeotrekTouristicEventParser"), + } + + def __init__(self, progress_cb=None, user=None, encoding='utf8'): + self.progress_cb = progress_cb + self.user = user + self.encoding = encoding + self.line = 0 + self.nb_success = 0 + self.nb_created = 0 + self.nb_updated = 0 + self.nb_unmodified = 0 + self.progress_cb = progress_cb + self.warnings = {} + self.report_by_api_v2_by_type = {} + + def parse(self, filename=None, limit=None): + if not os.path.exists(filename): + raise ImportError(_("This file doesn't exist")) + with open(filename, mode='r') as f: + json_aggregator = json.load(f) + + for key, datas in json_aggregator.items(): + self.report_by_api_v2_by_type[key] = {} + if not datas.get('data_to_import'): + raise + for model in datas['data_to_import']: + module_name, class_name = self.mapping_model_parser[model] + module = importlib.import_module(module_name) + parser = getattr(module, class_name) + Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas['portals'], + mapping=datas['mapping']) + Parser.parse() + self.report_by_api_v2_by_type[key][model] = { + 'nb_lines': Parser.line, + 'nb_success': Parser.nb_success, + 'nb_created': Parser.nb_created, + 'nb_updated': Parser.nb_updated, + 'nb_deleted': len(Parser.to_delete) if Parser.delete else None, + 'nb_unmodified': Parser.nb_unmodified, + 'warnings': Parser.warnings + } + + def report(self, output_format='txt'): + context = {'report': self.report_by_api_v2_by_type} + return render_to_string('common/parser_report_aggregator.{output_format}'.format(output_format=output_format), context) + + class GeotrekParser(AttachmentParserMixin, Parser): """ url_categories: url of the categories in api v2 (example: 'category': '/api/v2/touristiccontent_category/') @@ -903,6 +962,8 @@ class GeotrekParser(AttachmentParserMixin, Parser): categories_keys_api_v2: Key in the route of the category (example: /api/v2/touristiccontent_category/) corresponding to the model field eid_prefix: Prefix of your eid which allow to differentiate multiple GeotrekParser portals_filter: Portals which will be use for filter in api v2 (default: No portal filter) + mapping: Mapping between values in categories (example: /api/v2/touristiccontent_category/) and final values + Can be use when you want to change a value from the api/v2 """ model = None next_url = '' @@ -924,15 +985,18 @@ class GeotrekParser(AttachmentParserMixin, Parser): bbox = None eid_prefix = '' portals_filter = None + mapping = {} - def __init__(self, url=None, portals_filter=None, *args, **kwargs): + def __init__(self, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): super().__init__(*args, **kwargs) self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID self.bbox.transform(4326) # WGS84 self.portals_filter = portals_filter self.url = url if url else self.url - self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') + self.mapping = mapping if mapping else self.mapping + self.eid_prefix = eid_prefix if eid_prefix else self.eid_prefix + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not (f.name == 'id' or f.name == 'uuid')) self.m2m_fields = { f.name: f.name for f in self.model._meta.many_to_many @@ -945,10 +1009,9 @@ def __init__(self, url=None, portals_filter=None, *args, **kwargs): self.m2m_fields[key] = value self.translated_fields = [field for field in get_translated_fields(self.model)] # Generate a mapping dictionnary between id and the related label - for category in self.url_categories.keys(): + for category, route in self.url_categories.items(): if self.categories_keys_api_v2.get(category): - route = self.url_categories[category] - response = self.request_or_retry(f"{self.url}{route}") + response = self.request_or_retry(f"{self.url}/api/v2/{route}") self.field_options.setdefault(category, {}) self.field_options[category]["mapping"] = {} results = response.json()['results'] @@ -958,14 +1021,20 @@ def __init__(self, url=None, portals_filter=None, *args, **kwargs): label = result[self.categories_keys_api_v2[category]] if isinstance(result[self.categories_keys_api_v2[category]], dict): if label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE]: - self.field_options[category]["mapping"][id_result] = label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + self.field_options[category]["mapping"][id_result] = self.replace_mapping(label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE], route) else: if label: - self.field_options[category]["mapping"][id_result] = label + self.field_options[category]["mapping"][id_result] = self.replace_mapping(label, category) else: raise ImproperlyConfigured(f"{category} is not configured in categories_keys_api_v2") self.creator, created = get_user_model().objects.get_or_create(username='import', defaults={'is_active': False}) + def replace_mapping(self, label, route): + for key, list_map in self.mapping.get(route, {}).items(): + if label in list_map: + return key + return label + def start(self): super().start() kwargs = self.get_to_delete_kwargs() @@ -1007,12 +1076,12 @@ def next_row(self): Geotrek API is paginated, run until "next" is empty :returns row """ + portals = self.portals_filter + params = { + 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), + 'portals': ','.join(portals) if portals else '' + } while self.next_url: - portals = self.portals_filter - params = { - 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'portals': ','.join(portals) if portals else '' - } response = self.request_or_retry(self.next_url, params=params) self.root = response.json() self.nb = int(self.root['count']) diff --git a/geotrek/common/templates/common/parser_report_aggregator.txt b/geotrek/common/templates/common/parser_report_aggregator.txt new file mode 100644 index 0000000000..8b4dbd127f --- /dev/null +++ b/geotrek/common/templates/common/parser_report_aggregator.txt @@ -0,0 +1,8 @@ +{% for key, value in report.items %} +{{ key }} : +______________________________________________ +{% for model, report_by_model in value.items %} +{{ model }} : +{% include "common/parser_report.txt" with nb_success=report_by_model.nb_success nb_created=report_by_model.nb_created nb_deleted=report_by_model.nb_deleted nb_updated=report_by_model.nb_updated nb_lines=report_by_model.nb_lines nb_unmodified=report_by_model.nb_unmodified warnings=report_by_model.warnings %} +{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py index 9eee9e7a91..cedc2c6c59 100644 --- a/geotrek/infrastructure/parsers.py +++ b/geotrek/infrastructure/parsers.py @@ -16,8 +16,8 @@ class GeotrekInfrastructureParser(GeotrekParser): "geom": "geometry" } url_categories = { - 'condition': '/api/v2/infrastructure_condition/', - 'type': '/api/v2/infrastructure_type/', + 'condition': 'infrastructure_condition', + 'type': 'infrastructure_type', } categories_keys_api_v2 = { 'condition': 'label', diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py index 5720c80f67..720448abc5 100644 --- a/geotrek/signage/parsers.py +++ b/geotrek/signage/parsers.py @@ -16,9 +16,9 @@ class GeotrekSignageParser(GeotrekParser): "geom": "geometry" } url_categories = { - 'sealing': '/api/v2/signage_sealing/', - 'condition': '/api/v2/infrastructure_condition/', - 'type': '/api/v2/signage_type/', + 'sealing': 'signage_sealing', + 'condition': 'infrastructure_condition', + 'type': 'signage_type', } categories_keys_api_v2 = { 'condition': 'label', diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index a2796f48a6..a303491c1a 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -911,8 +911,8 @@ class GeotrekTouristicContentParser(GeotrekParser): } url_categories = { - "category": "/api/v2/touristiccontent_category/", - "themes": "/api/v2/theme/", + "category": "touristiccontent_category", + "themes": "theme", } categories_keys_api_v2 = { @@ -983,7 +983,7 @@ class GeotrekTouristicEventParser(GeotrekParser): "geom": "geometry" } url_categories = { - "type": "/api/v2/touristicevent_type/", + "type": "touristicevent_type", } categories_keys_api_v2 = { 'type': 'type', @@ -1003,7 +1003,7 @@ class GeotrekInformationDeskParser(GeotrekParser): model = InformationDesk constant_fields = {} replace_fields = { - "uuid": ["uuid", "id"], + "eid": "id", "geom": ["latitude", "longitude"], "photo": "photo_url" } diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 22fa044bd5..93d72c9c75 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -79,13 +79,13 @@ class GeotrekTrekParser(GeotrekParser): "geom": "geometry" } url_categories = { - "difficulty": "/api/v2/trek_difficulty/", - "route": "/api/v2/trek_route/", - "themes": "/api/v2/theme/", - "practice": "/api/v2/trek_practice/", - "accessibilities": "/api/v2/trek_accessibility/", - "networks": "/api/v2/trek_network/", - 'labels': '/api/v2/label/' + "difficulty": "trek_difficulty", + "route": "trek_route", + "themes": "theme", + "practice": "trek_practice", + "accessibilities": "trek_accessibility", + "networks": "trek_network", + 'labels': 'label' } categories_keys_api_v2 = { 'difficulty': 'label', @@ -167,7 +167,7 @@ class GeotrekServiceParser(GeotrekParser): "geom": "geometry" } url_categories = { - "type": "/api/v2/service_type/", + "type": "service_type", } categories_keys_api_v2 = { 'type': 'name', @@ -195,7 +195,7 @@ class GeotrekPOIParser(GeotrekParser): "geom": "geometry" } url_categories = { - "type": "/api/v2/poi_type/", + "type": "poi_type", } categories_keys_api_v2 = { 'type': 'label', From 2d6fdfafec8761b806680c5aabeb0d98fd65dd0e Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 5 Aug 2022 14:41:09 +0200 Subject: [PATCH 043/116] Add Signage Infrastructure Parser --- geotrek/common/parsers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 2e30b62680..b04f0321cb 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -907,6 +907,8 @@ class GeotrekAggregatorParser: "InformationDesk": ("geotrek.tourism.parsers", "GeotrekInformationDeskParser"), "TouristicContent": ("geotrek.tourism.parsers", "GeotrekTouristicContentParser"), "TouristicEvent": ("geotrek.tourism.parsers", "GeotrekTouristicEventParser"), + "Signage": ("geotrek.signage.parsers", "GeotrekSignageParser"), + "Infrastructure": ("geotrek.infrastructure.parsers", "GeotrekInfrastructureParser"), } def __init__(self, progress_cb=None, user=None, encoding='utf8'): From 795d0ec6f7baa9df1de8e81b3f93cb1cbf3b22ae Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 5 Aug 2022 15:38:58 +0200 Subject: [PATCH 044/116] Add license trek --- geotrek/common/parsers.py | 38 ++++++++++++++++++- .../tests/data/geotrek_parser_v2/trek.json | 2 +- geotrek/trekking/tests/test_parsers.py | 4 +- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index b04f0321cb..8fe1b4608a 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -34,9 +34,10 @@ from paperclip.models import attachment_upload from geotrek.authent.models import default_structure -from geotrek.common.models import FileType, Attachment +from geotrek.common.models import FileType, Attachment, License from geotrek.common.utils.translation import get_translated_fields + if 'modeltranslation' in settings.INSTALLED_APPS: from modeltranslation.fields import TranslationField @@ -1037,6 +1038,39 @@ def replace_mapping(self, label, route): return key return label + def generate_attachments(self, src, val, attachments_to_delete, updated): + attachments = [] + for url, legend, author, license in self.filter_attachments(src, val): + url = self.base_url + url + legend = legend or "" + author = author or "" + license = License.objects.get_or_create(label=license)[0] if license else None + basename, ext = os.path.splitext(os.path.basename(url)) + name = '%s%s' % (basename[:128], ext) + found, updated = self.check_attachment_updated(attachments_to_delete, updated, name=name, url=url, + legend=legend, author=author) + if found: + continue + + parsed_url = urlparse(url) + attachment = self.generate_attachment(author=author, legend=legend, license=license) + save, updated = self.generate_content_attachment(attachment, parsed_url, url, updated, name) + if not save: + continue + attachments.append(attachment) + updated = True + return updated, attachments + + def generate_attachment(self, **kwargs): + attachment = Attachment() + attachment.content_object = self.obj + attachment.filetype = self.filetype + attachment.creator = self.creator + attachment.author = kwargs.get('author') + attachment.legend = textwrap.shorten(kwargs.get('legend'), width=127) + attachment.license = kwargs.get('license') + return attachment + def start(self): super().start() kwargs = self.get_to_delete_kwargs() @@ -1050,7 +1084,7 @@ def filter_eid(self, src, val): return f'{self.eid_prefix}{val}' def filter_attachments(self, src, val): - return [(subval.get('url'), subval.get('legend'), subval.get('author')) for subval in val] + return [(subval.get('url'), subval.get('legend'), subval.get('author'), subval.get('license')) for subval in val] def apply_filter(self, dst, src, val): val = super().apply_filter(dst, src, val) diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json index 553595d5aa..fe571cf486 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -82,7 +82,7 @@ "backend": "", "type": "image", "author": "Grégory Tonon", - "license": null, + "license": "License", "thumbnail": "https://foo.fr/media/paperclip/trekking_trek/2/1024px-les_pyrenees_depuis_le_pic_des_3_seigneurs.jpg.400x0_q85.jpg", "legend": "Vue sur la chaîne des Pyrénées, depuis le pics des 3 Seigneurs", "title": "1024px-Les_Pyrénées_depuis_le_pic_des_3_Seigneurs", diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index ed899302c4..d99afa9125 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -10,7 +10,7 @@ from django.test import TestCase from django.test.utils import override_settings -from geotrek.common.models import Theme, FileType +from geotrek.common.models import Theme, FileType, Attachment from geotrek.trekking.models import POI, Service, Trek, DifficultyLevel, Route from geotrek.trekking.parsers import TrekParser, GeotrekPOIParser, GeotrekServiceParser, GeotrekTrekParser @@ -212,6 +212,8 @@ def mocked_json(): self.assertEqual(trek.children.first().name, "Foo") self.assertEqual(trek.labels.count(), 3) self.assertEqual(trek.labels.first().name, "Chien autorisé") + self.assertEqual(Attachment.objects.filter(object_id=trek.pk).count(), 3) + self.assertEqual(Attachment.objects.get(object_id=trek.pk, license__isnull=False).license.label, "License") @mock.patch('requests.get') @mock.patch('requests.head') From 0d93cf9f0b8a718de2cd7c61194bc955460128f9 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 5 Aug 2022 15:39:32 +0200 Subject: [PATCH 045/116] Add test erro children trek api parser v2 --- geotrek/trekking/parsers.py | 2 +- .../trek_children_do_not_exist.json | 30 +++++++++++++++++++ .../trek_wrong_children.json | 18 ++++------- geotrek/trekking/tests/test_parsers.py | 28 ++++++++++++++++- 4 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 93d72c9c75..6b3a1e585c 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -151,7 +151,7 @@ def end(self): order=order) order += 1 except Exception as e: - self.add_warning(_(f"An error occured in children generation : {e}")) + self.add_warning(_(f"An error occured in children generation : {getattr(e, 'message', repr(e))}")) class GeotrekServiceParser(GeotrekParser): diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json new file mode 100644 index 0000000000..6fa8b5887c --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json @@ -0,0 +1,30 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 2, + "children": [ + 10439, + 10888 + ] + }, + { + "id": 10443, + "children": [] + }, + { + "id": 2849, + "children": [] + }, + { + "id": 10439, + "children": [] + }, + { + "id": 8700, + "children": [] + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json index 6fa8b5887c..0c78b12976 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json @@ -4,27 +4,19 @@ "previous": null, "results": [ { - "id": 2, - "children": [ - 10439, - 10888 - ] + "id": 2 }, { - "id": 10443, - "children": [] + "id": 10443 }, { - "id": 2849, - "children": [] + "id": 2849 }, { - "id": 10439, - "children": [] + "id": 10439 }, { - "id": 8700, - "children": [] + "id": 8700 } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index d99afa9125..0aaf506176 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -252,6 +252,31 @@ def mocked_json(): call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrek2TrekParser', verbosity=0) self.assertEqual(Trek.objects.count(), 6) + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_children_do_not_exist(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', + 'trek_children_do_not_exist.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + output = StringIO() + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, + stdout=output) + self.assertIn("One trek has not be generated for Boucle du Pic des Trois Seigneurs", output.getvalue()) + @mock.patch('requests.get') @mock.patch('requests.head') def test_wrong_children_error(self, mocked_head, mocked_get): @@ -273,9 +298,10 @@ def mocked_json(): mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 output = StringIO() + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, stdout=output) - self.assertIn("An error occured in children generation", output.getvalue()) + self.assertIn("An error occured in children generation : KeyError('children')", output.getvalue()) @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') From 96be0b26b211fc5a5f7a790b2b7348f33e58cd61 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 5 Aug 2022 19:21:19 +0200 Subject: [PATCH 046/116] Add create categories, add mapping tourism --- geotrek/common/parsers.py | 20 ++++++++++++++++---- geotrek/infrastructure/parsers.py | 4 ++-- geotrek/signage/parsers.py | 4 ++-- geotrek/tourism/parsers.py | 18 +++++++++--------- geotrek/trekking/parsers.py | 12 ++++++------ 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 8fe1b4608a..ed2a2bb091 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -940,7 +940,7 @@ def parse(self, filename=None, limit=None): module = importlib.import_module(module_name) parser = getattr(module, class_name) Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas['portals'], - mapping=datas['mapping']) + mapping=datas['mapping'], create_categories=datas['create']) Parser.parse() self.report_by_api_v2_by_type[key][model] = { 'nb_lines': Parser.line, @@ -967,6 +967,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): portals_filter: Portals which will be use for filter in api v2 (default: No portal filter) mapping: Mapping between values in categories (example: /api/v2/touristiccontent_category/) and final values Can be use when you want to change a value from the api/v2 + create_categories: Create all categories during importation """ model = None next_url = '' @@ -989,8 +990,9 @@ class GeotrekParser(AttachmentParserMixin, Parser): eid_prefix = '' portals_filter = None mapping = {} + create_categories = False - def __init__(self, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): + def __init__(self, create_categories=None, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): super().__init__(*args, **kwargs) self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID @@ -999,6 +1001,7 @@ def __init__(self, eid_prefix=None, mapping=None, portals_filter=None, url=None, self.url = url if url else self.url self.mapping = mapping if mapping else self.mapping self.eid_prefix = eid_prefix if eid_prefix else self.eid_prefix + self.create_categories = create_categories if create_categories else self.create_categories self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not (f.name == 'id' or f.name == 'uuid')) self.m2m_fields = { f.name: f.name @@ -1017,6 +1020,8 @@ def __init__(self, eid_prefix=None, mapping=None, portals_filter=None, url=None, response = self.request_or_retry(f"{self.url}/api/v2/{route}") self.field_options.setdefault(category, {}) self.field_options[category]["mapping"] = {} + if self.create_categories: + self.field_options[category]["create"] = True results = response.json()['results'] # for element in category url map the id with its label for result in results: @@ -1075,10 +1080,16 @@ def start(self): super().start() kwargs = self.get_to_delete_kwargs() kwargs['eid__startswith'] = self.eid_prefix + params = { + 'fields': 'id', + 'page_size': 10000 + } + response = self.request_or_retry(self.next_url, params=params) + ids = [element['id'] for element in response.json()['results']] if kwargs is None: self.to_delete = set() else: - self.to_delete = set(self.model.objects.filter(**kwargs).values_list('pk', flat=True)) + self.to_delete = set(self.model.objects.filter(**kwargs).exclude(eid__in=ids).values_list('pk', flat=True)) def filter_eid(self, src, val): return f'{self.eid_prefix}{val}' @@ -1115,7 +1126,8 @@ def next_row(self): portals = self.portals_filter params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'portals': ','.join(portals) if portals else '' + 'portals': ','.join(portals) if portals else '', + 'updated_after': self.model.objects.latest('date_update').date_update.strftime('%Y-%m-%d') if self.model.objects.exists() and 'date_update' in self.model._meta.get_fields() else None } while self.next_url: response = self.request_or_retry(self.next_url, params=params) diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py index cedc2c6c59..fd1f7b7319 100644 --- a/geotrek/infrastructure/parsers.py +++ b/geotrek/infrastructure/parsers.py @@ -28,6 +28,6 @@ class GeotrekInfrastructureParser(GeotrekParser): 'type': 'label' } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/infrastructure" - return super().next_row() diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py index 720448abc5..4e6e728480 100644 --- a/geotrek/signage/parsers.py +++ b/geotrek/signage/parsers.py @@ -31,6 +31,6 @@ class GeotrekSignageParser(GeotrekParser): 'type': 'label' } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/signage" - return super().next_row() diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index a303491c1a..dcff986b5b 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -945,14 +945,14 @@ def __init__(self, *args, **kwargs): for type_category in r['types']: label_lang = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] id_category = type_category["id"] + if self.create_categories: + self.field_options['type1']["create"] = True + self.field_options['type2']["create"] = True if id_category % 10 == 1: - self.field_options['type1']["mapping"][id_category] = label_lang if label_lang else None + self.field_options['type1']["mapping"][id_category] = self.replace_mapping(label_lang, 'type1') if label_lang else None if id_category % 10 == 2: - self.field_options['type2']["mapping"][id_category] = label_lang if label_lang else None - - def next_row(self): + self.field_options['type2']["mapping"][id_category] = self.replace_mapping(label_lang, 'type2') if label_lang else None self.next_url = f"{self.url}/api/v2/touristiccontent" - return super().next_row() def filter_type1(self, src, val): type1_result = [] @@ -992,9 +992,9 @@ class GeotrekTouristicEventParser(GeotrekParser): 'type': 'type', } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/touristicevent" - return super().next_row() class GeotrekInformationDeskParser(GeotrekParser): @@ -1018,9 +1018,9 @@ class GeotrekInformationDeskParser(GeotrekParser): 'geom': {'required': True}, } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/informationdesk" - return super().next_row() def filter_uuid(self, src, val): uuid, id_iddesk = val diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 6b3a1e585c..d37fad3655 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -106,9 +106,9 @@ class GeotrekTrekParser(GeotrekParser): 'labels': 'name' } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/trek" - return super().next_row() def filter_parking_location(self, src, val): if val: @@ -176,9 +176,9 @@ class GeotrekServiceParser(GeotrekParser): 'type': 'name' } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/service" - return super().next_row() class GeotrekPOIParser(GeotrekParser): @@ -204,6 +204,6 @@ class GeotrekPOIParser(GeotrekParser): 'type': 'label', } - def next_row(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/poi" - return super().next_row() From c422957e05d721709a5d5ec378dd3db8f32f84c5 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Sun, 7 Aug 2022 15:10:48 +0200 Subject: [PATCH 047/116] Fix linting init geotrekparser --- geotrek/common/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index ed2a2bb091..37f5eea4f7 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -992,7 +992,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): mapping = {} create_categories = False - def __init__(self, create_categories=None, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): + def __init__(self, create_categories=None, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): super().__init__(*args, **kwargs) self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID From 1c1aacebb1021b692bd8ba732875b4ccbffcc866 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Sun, 7 Aug 2022 16:36:07 +0200 Subject: [PATCH 048/116] Fix tests with ids for deletion parser api v2 --- .../geotrek_parser_v2/infrastructure_ids.json | 13 +++++++++++ geotrek/infrastructure/tests/test_parsers.py | 3 ++- .../data/geotrek_parser_v2/signage_ids.json | 13 +++++++++++ geotrek/signage/tests/test_parsers.py | 3 ++- .../informationdesk_ids.json | 13 +++++++++++ .../touristiccontent_ids.json | 13 +++++++++++ .../geotrek_parser_v2/touristicevent_ids.json | 13 +++++++++++ geotrek/tourism/tests/test_parsers.py | 5 ++++- .../tests/data/geotrek_parser_v2/poi_ids.json | 13 +++++++++++ .../data/geotrek_parser_v2/service_ids.json | 13 +++++++++++ .../data/geotrek_parser_v2/trek_ids.json | 22 +++++++++++++++++++ geotrek/trekking/tests/test_parsers.py | 19 ++++++++-------- 12 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json create mode 100644 geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json create mode 100644 geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json diff --git a/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json new file mode 100644 index 0000000000..220f595ded --- /dev/null +++ b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 2898 + }, + { + "id": 8385 + } + ] +} \ No newline at end of file diff --git a/geotrek/infrastructure/tests/test_parsers.py b/geotrek/infrastructure/tests/test_parsers.py index b252d8287c..9f8413a8af 100644 --- a/geotrek/infrastructure/tests/test_parsers.py +++ b/geotrek/infrastructure/tests/test_parsers.py @@ -31,7 +31,8 @@ def setUpTestData(cls): @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") def test_create(self, mocked_head, mocked_get): self.mock_time = 0 - self.mock_json_order = ['infrastructure_condition.json', 'infrastructure_type.json', 'infrastructure.json', ] + self.mock_json_order = ['infrastructure_condition.json', 'infrastructure_type.json', 'infrastructure_ids.json', + 'infrastructure.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', diff --git a/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json b/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json new file mode 100644 index 0000000000..2b648faf7a --- /dev/null +++ b/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 8416 + }, + { + "id": 8455 + } + ] +} \ No newline at end of file diff --git a/geotrek/signage/tests/test_parsers.py b/geotrek/signage/tests/test_parsers.py index ef8816b39c..0a912f94ea 100644 --- a/geotrek/signage/tests/test_parsers.py +++ b/geotrek/signage/tests/test_parsers.py @@ -32,7 +32,8 @@ def setUpTestData(cls): @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") def test_create(self, mocked_head, mocked_get): self.mock_time = 0 - self.mock_json_order = ['signage_sealing.json', 'signage_condition.json', 'signage_type.json', 'signage.json', ] + self.mock_json_order = ['signage_sealing.json', 'signage_condition.json', 'signage_type.json', 'signage_ids.json', + 'signage.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json new file mode 100644 index 0000000000..20ea9e4cef --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 1 + }, + { + "id": 2 + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json new file mode 100644 index 0000000000..5e51ae2673 --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 10 + }, + { + "id": 22 + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json new file mode 100644 index 0000000000..a935933a50 --- /dev/null +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 3 + }, + { + "id": 1 + } + ] +} \ No newline at end of file diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index c79a2c793f..cd52fcbc2f 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -693,6 +693,7 @@ def test_create(self, mocked_head, mocked_get): self.mock_json_order = ['touristiccontent_category.json', 'touristiccontent_themes.json', 'touristiccontent_category.json', + 'touristiccontent_ids.json', 'touristiccontent.json'] def mocked_json(): @@ -728,6 +729,7 @@ def setUpTestData(cls): def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['touristicevent_type.json', + 'touristicevent_ids.json', 'touristicevent.json'] def mocked_json(): @@ -762,7 +764,8 @@ def setUpTestData(cls): @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") def test_create(self, mocked_download_attachment, mocked_get): self.mock_time = 0 - self.mock_json_order = ['informationdesk.json', ] + self.mock_json_order = ['informationdesk_ids.json', + 'informationdesk.json', ] self.mock_download = 0 def mocked_json(): diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json new file mode 100644 index 0000000000..962735c479 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 2867 + }, + { + "id": 2869 + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json new file mode 100644 index 0000000000..47cfe79b80 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 4886 + }, + { + "id": 10393 + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json new file mode 100644 index 0000000000..0c78b12976 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json @@ -0,0 +1,22 @@ +{ + "count": 5, + "next": null, + "previous": null, + "results": [ + { + "id": 2 + }, + { + "id": 10443 + }, + { + "id": 2849 + }, + { + "id": 10439 + }, + { + "id": 8700 + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 0aaf506176..59f9e2665a 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -186,7 +186,8 @@ def setUpTestData(cls): def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', - 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', 'trek_children.json', ] + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', + 'trek.json', 'trek_children.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -220,10 +221,10 @@ def mocked_json(): def test_create_multiple(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', - 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', 'trek_children.json', 'trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', - 'trek_2.json', 'trek_children.json', ] + 'trek_ids.json', 'trek_2.json', 'trek_children.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -257,8 +258,8 @@ def mocked_json(): def test_children_do_not_exist(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', - 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', - 'trek_children_do_not_exist.json', ] + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', + 'trek.json', 'trek_children_do_not_exist.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -282,8 +283,8 @@ def mocked_json(): def test_wrong_children_error(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', - 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek.json', - 'trek_wrong_children.json', ] + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', + 'trek.json', 'trek_wrong_children.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -315,7 +316,7 @@ def setUpTestData(cls): @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") def test_create(self, mocked_head, mocked_get): self.mock_time = 0 - self.mock_json_order = ['poi_type.json', 'poi.json'] + self.mock_json_order = ['poi_type.json', 'poi_ids.json', 'poi.json'] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -350,7 +351,7 @@ def setUpTestData(cls): @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") def test_create(self, mocked_head, mocked_get): self.mock_time = 0 - self.mock_json_order = ['service_type.json', 'service.json'] + self.mock_json_order = ['service_type.json', 'service_ids.json', 'service.json'] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', From b7260c3ec0d9a983fbdd6f5b9ac2f989ddd9407c Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Sun, 7 Aug 2022 19:09:43 +0200 Subject: [PATCH 049/116] Fix test repr not always the same --- geotrek/trekking/tests/test_parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 59f9e2665a..5b653d85fc 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -302,7 +302,7 @@ def mocked_json(): call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, stdout=output) - self.assertIn("An error occured in children generation : KeyError('children')", output.getvalue()) + self.assertIn("An error occured in children generation : KeyError('children'", output.getvalue()) @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') From e7a161a5c2bb3a756cbaca466c81619198e35916 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 11:58:02 +0200 Subject: [PATCH 050/116] Add warning with dynamic segmenation model not possible to import --- geotrek/common/parsers.py | 41 +++++++++++++++++++--------- geotrek/common/tests/test_parsers.py | 13 ++++++++- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 37f5eea4f7..b05bcdf294 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -912,6 +912,8 @@ class GeotrekAggregatorParser: "Infrastructure": ("geotrek.infrastructure.parsers", "GeotrekInfrastructureParser"), } + invalid_model_topology = ['Trek', 'POI', 'Service', 'Signage', 'Infrastructure'] + def __init__(self, progress_cb=None, user=None, encoding='utf8'): self.progress_cb = progress_cb self.user = user @@ -925,6 +927,11 @@ def __init__(self, progress_cb=None, user=None, encoding='utf8'): self.warnings = {} self.report_by_api_v2_by_type = {} + def add_warning(self, msg, model): + key = _(f"Model {model}") + warnings = self.warnings.setdefault(key, []) + warnings.append(msg) + def parse(self, filename=None, limit=None): if not os.path.exists(filename): raise ImportError(_("This file doesn't exist")) @@ -936,20 +943,28 @@ def parse(self, filename=None, limit=None): if not datas.get('data_to_import'): raise for model in datas['data_to_import']: - module_name, class_name = self.mapping_model_parser[model] - module = importlib.import_module(module_name) - parser = getattr(module, class_name) - Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas['portals'], - mapping=datas['mapping'], create_categories=datas['create']) - Parser.parse() + if settings.TREKKING_TOPOLOGY_ENABLED: + if model in self.invalid_model_topology: + warning = f"{model}s can't be imported with dynamic segmentation" + logger.warning(warning) + self.add_warning(warning, model) + Parser = None + else: + module_name, class_name = self.mapping_model_parser[model] + module = importlib.import_module(module_name) + parser = getattr(module, class_name) + Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas['portals'], + mapping=datas['mapping'], create_categories=datas['create']) + Parser.parse() + self.report_by_api_v2_by_type[key][model] = { - 'nb_lines': Parser.line, - 'nb_success': Parser.nb_success, - 'nb_created': Parser.nb_created, - 'nb_updated': Parser.nb_updated, - 'nb_deleted': len(Parser.to_delete) if Parser.delete else None, - 'nb_unmodified': Parser.nb_unmodified, - 'warnings': Parser.warnings + 'nb_lines': Parser.line if Parser else 0, + 'nb_success': Parser.nb_success if Parser else 0, + 'nb_created': Parser.nb_created if Parser else 0, + 'nb_updated': Parser.nb_updated if Parser else 0, + 'nb_deleted': len(Parser.to_delete) if Parser and Parser.delete else None, + 'nb_unmodified': Parser.nb_unmodified if Parser else 0, + 'warnings': Parser.warnings if Parser else self.warnings } def report(self, output_format='txt'): diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 853710e0eb..f071a10112 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -1,5 +1,5 @@ import os -from unittest import mock +from unittest import mock, skipIf from shutil import rmtree from tempfile import mkdtemp from io import StringIO @@ -489,3 +489,14 @@ def setUp(self, *args, **kwargs): def test_improperly_configurated_categories(self): with self.assertRaisesRegex(ImproperlyConfigured, 'foo_field is not configured in categories_keys_api_v2'): call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestParser', verbosity=2) + + @skipIf(not settings.TREKKING_TOPOLOGY_ENABLED, 'Test with dynamic segmentation only') + def test_geotrek_aggregator_parser_model_dynamic_segmentation(self): + output = StringIO() + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', 'config_aggregator_ds.json') + call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, + stdout=output) + string_parser = output.getvalue() + self.assertIn("Services can't be imported with dynamic segmentation", string_parser) + self.assertIn("POIs can't be imported with dynamic segmentation", string_parser) + self.assertIn("Treks can't be imported with dynamic segmentation", string_parser) From 380e7a3d3fdcde77056ff30e48e6605db480a49c Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 11:58:39 +0200 Subject: [PATCH 051/116] Avoid error request without results key in json --- geotrek/common/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index b05bcdf294..acce70ad70 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1037,7 +1037,7 @@ def __init__(self, create_categories=None, eid_prefix=None, mapping=None, portal self.field_options[category]["mapping"] = {} if self.create_categories: self.field_options[category]["create"] = True - results = response.json()['results'] + results = response.json().get('results', []) # for element in category url map the id with its label for result in results: id_result = result['id'] @@ -1100,7 +1100,7 @@ def start(self): 'page_size': 10000 } response = self.request_or_retry(self.next_url, params=params) - ids = [element['id'] for element in response.json()['results']] + ids = [element['id'] for element in response.json().get('results', [])] if kwargs is None: self.to_delete = set() else: From bf6f026b758c7c1e1b32906202a8e0fbe1f80f50 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 12:01:20 +0200 Subject: [PATCH 052/116] Add file test config aggregator with dynamic segmentation --- .../config_aggregator_ds.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_ds.json diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_ds.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_ds.json new file mode 100644 index 0000000000..1ed8fcd956 --- /dev/null +++ b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_ds.json @@ -0,0 +1,30 @@ +{ + "URL_1": { + "url": "url_1", + "data_to_import": ["Service", "POI"], + "portals": null, + "mapping": { + "trek_practice": { + "Randonnée Pédestre": ["Pédestre"], + "Vélo": ["VTT", "Vélo"] + }, + "trek_difficulty": { + "Moyen": ["Facile"] + } + } + }, + "URL_2": { + "url": "url_2", + "data_to_import": ["Trek", "POI"], + "portals": null, + "mapping": { + "trek_practice": { + "Randonnée Pédestre": ["Pédestre"], + "Vélo": ["VTT", "Vélo"] + }, + "trek_difficulty": { + "Moyen": ["Facile"] + } + } + } +} From 79b7ed004d391c6510f4ff969ef873fbc84b3be1 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 14:55:43 +0200 Subject: [PATCH 053/116] Add test aggregator parser --- geotrek/common/parsers.py | 19 +++++--- .../geotrek_parser_v2/config_aggregator.json | 46 +++++++++++++++++++ geotrek/common/tests/test_parsers.py | 24 ++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index acce70ad70..2e908c8459 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -927,8 +927,7 @@ def __init__(self, progress_cb=None, user=None, encoding='utf8'): self.warnings = {} self.report_by_api_v2_by_type = {} - def add_warning(self, msg, model): - key = _(f"Model {model}") + def add_warning(self, key, msg): warnings = self.warnings.setdefault(key, []) warnings.append(msg) @@ -943,19 +942,25 @@ def parse(self, filename=None, limit=None): if not datas.get('data_to_import'): raise for model in datas['data_to_import']: + Parser = None if settings.TREKKING_TOPOLOGY_ENABLED: if model in self.invalid_model_topology: warning = f"{model}s can't be imported with dynamic segmentation" logger.warning(warning) - self.add_warning(warning, model) - Parser = None + key = _(f"Model {model}") + self.add_warning(key, warning) else: module_name, class_name = self.mapping_model_parser[model] module = importlib.import_module(module_name) parser = getattr(module, class_name) - Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas['portals'], - mapping=datas['mapping'], create_categories=datas['create']) - Parser.parse() + if 'url' not in datas: + warning = f"{key} has no url" + key = _("Geotrek-admin") + self.add_warning(key, warning) + else: + Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas.get('portals'), + mapping=datas.get('mapping'), create_categories=datas.get('create')) + Parser.parse() self.report_by_api_v2_by_type[key][model] = { 'nb_lines': Parser.line if Parser else 0, diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json new file mode 100644 index 0000000000..7b79a1e9c5 --- /dev/null +++ b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json @@ -0,0 +1,46 @@ +{ + "URL_1": { + "url": "URL_1", + "data_to_import": ["Trek", "POI"], + "portals": ["portal a", "portal b"], + "mapping": { + "trek_practice": { + "Randonnée Pédestre": ["Pédestre"], + "Vélo": ["VTT", "Vélo"] + }, + "trek_difficulty": { + "Moyen": ["Facile"] + } + } + }, + "URL_2": { + "url": "URL_2", + "data_to_import": ["Trek", "Service", "POI"], + "portals": null, + "mapping": { + "trek_practice": { + "Randonnée Pédestre": ["Pédestre"], + "Vélo": ["VTT", "Vélo"] + }, + "trek_difficulty": { + "Moyen": ["Facile"] + } + }, + "create": true + }, + "URL_3": { + "url": "URL_3", + "data_to_import": ["POI", "InformationDesk", "TouristicContent"], + "portals": null, + "mapping": { + "trek_practice": { + "Randonnée Pédestre": ["Pédestre"], + "Vélo": ["VTT", "Vélo"] + }, + "trek_difficulty": { + "Moyen": ["Facile"] + } + }, + "create": true + } +} diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index f071a10112..5625ebd495 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -500,3 +500,27 @@ def test_geotrek_aggregator_parser_model_dynamic_segmentation(self): self.assertIn("Services can't be imported with dynamic segmentation", string_parser) self.assertIn("POIs can't be imported with dynamic segmentation", string_parser) self.assertIn("Treks can't be imported with dynamic segmentation", string_parser) + + @mock.patch('geotrek.common.parsers.importlib.import_module', return_value=mock.MagicMock()) + @mock.patch('requests.get') + def test_geotrek_aggregator_parser(self, mocked_get, mocked_import_module): + def mocked_json(): + return {} + + mocked_get.json = mocked_json + mocked_get.return_value.status_code = 200 + mocked_get.return_value.content = b'' + output = StringIO() + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', 'config_aggregator.json') + call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, + stdout=output) + string_parser = output.getvalue() + + self.assertIn('URL_1 :', string_parser) + self.assertIn('URL_2 :', string_parser) + self.assertIn('URL_3 :', string_parser) + + # "VTT", "Vélo" + # "Trek", "Service", "POI" + # "POI", "InformationDesk", "TouristicContent" + self.assertEqual(8, mocked_import_module.call_count) From 23ce77570c40aea66baaf388837ba4faec71b3af Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 15:04:59 +0200 Subject: [PATCH 054/116] Fix warning parser, add skipif add test no url --- geotrek/common/parsers.py | 8 ++++---- geotrek/common/tests/test_parsers.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 2e908c8459..9561101f8a 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -947,16 +947,16 @@ def parse(self, filename=None, limit=None): if model in self.invalid_model_topology: warning = f"{model}s can't be imported with dynamic segmentation" logger.warning(warning) - key = _(f"Model {model}") - self.add_warning(key, warning) + key_warning = _(f"Model {model}") + self.add_warning(key_warning, warning) else: module_name, class_name = self.mapping_model_parser[model] module = importlib.import_module(module_name) parser = getattr(module, class_name) if 'url' not in datas: warning = f"{key} has no url" - key = _("Geotrek-admin") - self.add_warning(key, warning) + key_warning = _("Geotrek-admin") + self.add_warning(key_warning, warning) else: Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas.get('portals'), mapping=datas.get('mapping'), create_categories=datas.get('create')) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 5625ebd495..0f8cb9e973 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -501,6 +501,7 @@ def test_geotrek_aggregator_parser_model_dynamic_segmentation(self): self.assertIn("POIs can't be imported with dynamic segmentation", string_parser) self.assertIn("Treks can't be imported with dynamic segmentation", string_parser) + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') @mock.patch('geotrek.common.parsers.importlib.import_module', return_value=mock.MagicMock()) @mock.patch('requests.get') def test_geotrek_aggregator_parser(self, mocked_get, mocked_import_module): @@ -524,3 +525,13 @@ def mocked_json(): # "Trek", "Service", "POI" # "POI", "InformationDesk", "TouristicContent" self.assertEqual(8, mocked_import_module.call_count) + + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') + def test_geotrek_aggregator_parser_no_url(self): + output = StringIO() + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', 'config_aggregator_no_url.json') + call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, + stdout=output) + string_parser = output.getvalue() + + self.assertIn('URL_1 has no url', string_parser) From 8a998a0f2d6befbe5f219c1476e95f12b4e0052c Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 15:05:17 +0200 Subject: [PATCH 055/116] Add config aggregator no url file --- .../config_aggregator_no_url.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_url.json diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_url.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_url.json new file mode 100644 index 0000000000..1fe779a8f0 --- /dev/null +++ b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_url.json @@ -0,0 +1,15 @@ +{ + "URL_1": { + "data_to_import": ["Trek", "POI"], + "portals": ["portal a", "portal b"], + "mapping": { + "trek_practice": { + "Randonnée Pédestre": ["Pédestre"], + "Vélo": ["VTT", "Vélo"] + }, + "trek_difficulty": { + "Moyen": ["Facile"] + } + } + } +} From ceae3eedb07a80eebb1f69d00241c45a6198d7c8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 15:41:23 +0200 Subject: [PATCH 056/116] Fix test render to string --- geotrek/common/tests/test_parsers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 0f8cb9e973..6b7093f03b 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -503,24 +503,24 @@ def test_geotrek_aggregator_parser_model_dynamic_segmentation(self): @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') @mock.patch('geotrek.common.parsers.importlib.import_module', return_value=mock.MagicMock()) + @mock.patch('django.template.loader.render_to_string') @mock.patch('requests.get') - def test_geotrek_aggregator_parser(self, mocked_get, mocked_import_module): + def test_geotrek_aggregator_parser(self, mocked_get, mocked_render, mocked_import_module): def mocked_json(): return {} + def side_effect_render(file, context): + return 'Render' + mocked_get.json = mocked_json mocked_get.return_value.status_code = 200 mocked_get.return_value.content = b'' + mocked_render.side_effect = side_effect_render output = StringIO() filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', 'config_aggregator.json') call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, stdout=output) - string_parser = output.getvalue() - - self.assertIn('URL_1 :', string_parser) - self.assertIn('URL_2 :', string_parser) - self.assertIn('URL_3 :', string_parser) - + self.assertEqual(output.getvalue(), 'Render\n') # "VTT", "Vélo" # "Trek", "Service", "POI" # "POI", "InformationDesk", "TouristicContent" From 9644fba563061e54a93bf149204feed2b9b8785b Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 16:30:43 +0200 Subject: [PATCH 057/116] Add language geotrek parser --- geotrek/common/parsers.py | 3 +++ .../trekking/tests/data/geotrek_parser_v2/poi.json | 4 ++-- .../trekking/tests/data/geotrek_parser_v2/trek.json | 2 +- geotrek/trekking/tests/test_parsers.py | 11 ++++++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 9561101f8a..972e5519e0 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1121,6 +1121,9 @@ def apply_filter(self, dst, src, val): val = super().apply_filter(dst, src, val) if dst in self.translated_fields: if isinstance(val, dict): + for key, final_value in val.items(): + if key in settings.MODELTRANSLATION_LANGUAGES: + self.set_value(f'{dst}_{key}', src, final_value) val = val.get(settings.MODELTRANSLATION_DEFAULT_LANGUAGE) return val diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json index 06bea73028..a8d3b29f48 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json @@ -22,9 +22,9 @@ }, "name": { "fr": "Pic des Trois Seigneurs", - "en": "", + "en": "Peak of the Three Lords", "es": "", - "it": "" + "it": "Picco dei Tre Signori" }, "attachments": [ { diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json index fe571cf486..2f5b4955ba 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -197,7 +197,7 @@ "fr": "Boucle du Pic des Trois Seigneurs", "en": "Loop of the pic of 3 lords", "es": "", - "it": "" + "it": "Foo bar" }, "networks": [ 2 diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 5b653d85fc..1727c95c1e 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -206,6 +206,7 @@ def mocked_json(): self.assertEqual(Trek.objects.count(), 5) trek = Trek.objects.all().first() self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(trek.name_it, "Foo bar") self.assertEqual(str(trek.difficulty), 'Très facile') self.assertEqual(str(trek.practice), 'Cheval') self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) @@ -243,6 +244,7 @@ def mocked_json(): self.assertEqual(Trek.objects.count(), 5) trek = Trek.objects.all().first() self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(trek.name_en, "Boucle du Pic des Trois Seigneurs") self.assertEqual(str(trek.difficulty), 'Très facile') self.assertEqual(str(trek.practice), 'Cheval') self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) @@ -313,7 +315,7 @@ def setUpTestData(cls): @mock.patch('requests.get') @mock.patch('requests.head') - @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="en") def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['poi_type.json', 'poi_ids.json', 'poi.json'] @@ -334,8 +336,11 @@ def mocked_json(): call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekPOIParser', verbosity=0) self.assertEqual(POI.objects.count(), 2) poi = POI.objects.all().first() - self.assertEqual(poi.name, "Pic des Trois Seigneurs") - self.assertEqual(str(poi.type), 'Sommet') + self.assertEqual(poi.name, "Peak of the Three Lords") + self.assertEqual(poi.name_fr, "Pic des Trois Seigneurs") + self.assertEqual(poi.name_en, "Peak of the Three Lords") + self.assertEqual(poi.name_it, "Picco dei Tre Signori") + self.assertEqual(str(poi.type), 'Peak') self.assertAlmostEqual(poi.geom.x, 572298.7056448072, places=5) self.assertAlmostEqual(poi.geom.y, 6193580.839504813, places=5) From cd3fb8bafc97b0a90ba3c661b209b8863504933c Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 17:06:22 +0200 Subject: [PATCH 058/116] Fix update deletion was not working, add test --- geotrek/common/parsers.py | 2 +- geotrek/trekking/tests/test_parsers.py | 45 ++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 972e5519e0..2411627d38 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1105,7 +1105,7 @@ def start(self): 'page_size': 10000 } response = self.request_or_retry(self.next_url, params=params) - ids = [element['id'] for element in response.json().get('results', [])] + ids = [f"{self.eid_prefix}{element['id']}" for element in response.json().get('results', [])] if kwargs is None: self.to_delete = set() else: diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 1727c95c1e..42f44fdbd6 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -134,7 +134,7 @@ def test_create(self): class TestGeotrekTrekParser(GeotrekTrekParser): url = "https://test.fr" - + eid_prefix = 'geotrek1' field_options = { 'difficulty': {'create': True, }, 'route': {'create': True, }, @@ -225,7 +225,7 @@ def test_create_multiple(self, mocked_head, mocked_get): 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', 'trek_children.json', 'trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', - 'trek_ids.json', 'trek_2.json', 'trek_children.json', ] + 'trek_ids_2.json', 'trek_2.json', 'trek_children.json', ] def mocked_json(): filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', @@ -306,6 +306,47 @@ def mocked_json(): stdout=output) self.assertIn("An error occured in children generation : KeyError('children'", output.getvalue()) + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_updated(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', + 'trek_children.json', 'trek_difficulty.json', 'trek_route.json', 'trek_theme.json', + 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', + 'trek_ids_2.json', 'trek_2.json', 'trek_children.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 5) + trek = Trek.objects.all().first() + self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(trek.name_en, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(str(trek.difficulty), 'Très facile') + self.assertEqual(str(trek.practice), 'Cheval') + self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) + self.assertAlmostEqual(trek.geom[0][1], 6190964.893167565, places=5) + self.assertEqual(trek.children.first().name, "Foo") + self.assertEqual(trek.labels.count(), 3) + self.assertEqual(trek.labels.first().name, "Chien autorisé") + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + # Trek 2 is still in ids (trek_ids_2) => it's not removed + self.assertEqual(Trek.objects.count(), 2) + trek = Trek.objects.all().first() + self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class POIGeotrekParserTests(TestCase): From aabd34d884a0af2ee8599c194fc0383ae89c822f Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 17:06:32 +0200 Subject: [PATCH 059/116] Remove uuid filter --- geotrek/tourism/parsers.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index dcff986b5b..28cfb4f682 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -1022,13 +1022,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.next_url = f"{self.url}/api/v2/informationdesk" - def filter_uuid(self, src, val): - uuid, id_iddesk = val - final_value = uuid - if not uuid: - final_value = id_iddesk - return final_value - def filter_geom(self, src, val): lat, lng = val return Point(lng, lat, srid=settings.API_SRID).transform(settings.SRID, clone=True) From db516ba9067a0b554e8b18207d4de0e0adff4aeb Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 17:14:50 +0200 Subject: [PATCH 060/116] Add file trek ids 2 --- .../tests/data/geotrek_parser_v2/trek_ids_2.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json new file mode 100644 index 0000000000..1fdcf2dd15 --- /dev/null +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json @@ -0,0 +1,13 @@ +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 8702 + }, + { + "id": 2 + } + ] +} \ No newline at end of file From dc3fb30af42e8f3fa039a17487602a31c4aec080 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 17:21:57 +0200 Subject: [PATCH 061/116] Add test attachments tourism --- geotrek/tourism/tests/test_parsers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index cd52fcbc2f..c42ef0a49a 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -716,6 +716,7 @@ def mocked_json(): self.assertEqual(str(touristic_content.name), "Balad'âne") self.assertAlmostEqual(touristic_content.geom.x, 568112.6362873032, places=5) self.assertAlmostEqual(touristic_content.geom.y, 6196929.676669887, places=5) + self.assertEqual(Attachment.objects.count(), 3) class TouristicEventGeotrekParserTests(TestCase): @@ -779,7 +780,7 @@ def mocked_download(*args, **kwargs): if self.mock_download > 0: return None self.mock_download += 1 - return b'' + return b'boo' # Mock GET mocked_get.return_value.status_code = 200 @@ -793,3 +794,4 @@ def mocked_download(*args, **kwargs): self.assertAlmostEqual(information_desk.geom.x, 573013.9272605104, places=5) self.assertAlmostEqual(information_desk.geom.y, 6276967.321705549, places=5) self.assertEqual(str(information_desk.photo), '') + self.assertEqual(InformationDesk.objects.exclude(photo='').first().photo.read(), b'boo') From 609e3f4749dc0de66e6e50fe7f6ddcdbbc78afb0 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 18:12:36 +0200 Subject: [PATCH 062/116] Add progress aggregator parser --- geotrek/common/parsers.py | 6 ++++-- ...ggregator.json => config_aggregator_multiple_admin.json} | 0 2 files changed, 4 insertions(+), 2 deletions(-) rename geotrek/common/tests/data/geotrek_parser_v2/{config_aggregator.json => config_aggregator_multiple_admin.json} (100%) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 2411627d38..11eaa167b8 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -958,8 +958,10 @@ def parse(self, filename=None, limit=None): key_warning = _("Geotrek-admin") self.add_warning(key_warning, warning) else: - Parser = parser(eid_prefix=key, url=datas['url'], portals_filter=datas.get('portals'), - mapping=datas.get('mapping'), create_categories=datas.get('create')) + Parser = parser(progress_cb=self.progress_cb, eid_prefix=key, url=datas['url'], + portals_filter=datas.get('portals'), mapping=datas.get('mapping'), + create_categories=datas.get('create')) + self.progress_cb(0, 0, f'{model} ({key})') Parser.parse() self.report_by_api_v2_by_type[key][model] = { diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_multiple_admin.json similarity index 100% rename from geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json rename to geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_multiple_admin.json From ff866ad3c6820dac517e5dbdf127e7184bd4b379 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 18:13:03 +0200 Subject: [PATCH 063/116] Add test with aggregator working --- .../geotrek_parser_v2/config_aggregator.json | 26 +++++++++ geotrek/common/tests/test_parsers.py | 56 +++++++++++++++++-- 2 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json new file mode 100644 index 0000000000..9ed9af9877 --- /dev/null +++ b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator.json @@ -0,0 +1,26 @@ +{ + "URL_1": { + "url": "URL_1", + "data_to_import": [ + "Trek", + "POI" + ], + "mapping": { + "trek_practice": { + "Randonnée Pédestre": [ + "Pédestre" + ], + "Vélo": [ + "VTT", + "Vélo" + ] + }, + "trek_difficulty": { + "Moyen": [ + "Facile" + ] + } + }, + "create": true + } +} \ No newline at end of file diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 6b7093f03b..b3b3a45ec3 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -17,7 +17,7 @@ from django.template.exceptions import TemplateDoesNotExist from geotrek.authent.tests.factories import StructureFactory -from geotrek.trekking.models import Trek +from geotrek.trekking.models import POI, Trek from geotrek.common.models import Organism, FileType, Attachment from geotrek.common.parsers import ( ExcelParser, AttachmentParserMixin, TourInSoftParser, ValueImportError, DownloadImportError, @@ -505,7 +505,7 @@ def test_geotrek_aggregator_parser_model_dynamic_segmentation(self): @mock.patch('geotrek.common.parsers.importlib.import_module', return_value=mock.MagicMock()) @mock.patch('django.template.loader.render_to_string') @mock.patch('requests.get') - def test_geotrek_aggregator_parser(self, mocked_get, mocked_render, mocked_import_module): + def test_geotrek_aggregator_parser_multiple_admin(self, mocked_get, mocked_render, mocked_import_module): def mocked_json(): return {} @@ -517,10 +517,13 @@ def side_effect_render(file, context): mocked_get.return_value.content = b'' mocked_render.side_effect = side_effect_render output = StringIO() - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', 'config_aggregator.json') + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + 'config_aggregator_multiple_admin.json') call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, stdout=output) - self.assertEqual(output.getvalue(), 'Render\n') + stdout_parser = output.getvalue() + self.assertIn('Render\n', stdout_parser) + self.assertIn('0000: Trek (URL_1) (00%)', stdout_parser) # "VTT", "Vélo" # "Trek", "Service", "POI" # "POI", "InformationDesk", "TouristicContent" @@ -535,3 +538,48 @@ def test_geotrek_aggregator_parser_no_url(self): string_parser = output.getvalue() self.assertIn('URL_1 has no url', string_parser) + + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_geotrek_aggregator_parser(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', + 'trek_route.json', + 'trek_theme.json', + 'trek_practice.json', + 'trek_accessibility.json', + 'trek_network.json', + 'trek_label.json', + 'trek_ids.json', + 'trek.json', + 'trek_children.json', + 'poi_type.json', + 'poi_ids.json', + 'poi.json'] + + def mocked_json(): + filename = os.path.join('geotrek', 'trekking', 'tests', 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + + output = StringIO() + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', 'config_aggregator.json') + call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, + stdout=output) + string_parser = output.getvalue() + self.assertIn('0000: Trek (URL_1) (00%)', string_parser) + self.assertIn('0000: POI (URL_1) (00%)', string_parser) + self.assertIn('5/5 lines imported.', string_parser) + self.assertIn('2/2 lines imported.', string_parser) + self.assertEqual(Trek.objects.count(), 5) + self.assertEqual(POI.objects.count(), 2) From daa5f8555379eeb832041ef5c298ebc646ee6b1e Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 8 Aug 2022 18:38:26 +0200 Subject: [PATCH 064/116] Fix import json --- geotrek/common/tests/test_parsers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index b3b3a45ec3..c601e02151 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -3,6 +3,7 @@ from shutil import rmtree from tempfile import mkdtemp from io import StringIO +import json import requests from requests import Response import urllib From f4d998a8e9b38b46008a34294b4e8b8db0a40cbd Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 11:13:40 +0200 Subject: [PATCH 065/116] Fix updated after --- geotrek/common/parsers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 11eaa167b8..0ea01438ae 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1149,10 +1149,16 @@ def next_row(self): :returns row """ portals = self.portals_filter + updated_after = None + if self.model.objects.filter(eid__startswith=self.eid_prefix).exists() and 'date_update' in [field.name for + field in + self.model._meta.get_fields()]: + updated_after = self.model.objects.filter(eid__startswith=self.eid_prefix).latest('date_update').date_update.strftime('%Y-%m-%d') + params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), 'portals': ','.join(portals) if portals else '', - 'updated_after': self.model.objects.latest('date_update').date_update.strftime('%Y-%m-%d') if self.model.objects.exists() and 'date_update' in self.model._meta.get_fields() else None + 'updated_after': updated_after } while self.next_url: response = self.request_or_retry(self.next_url, params=params) From de53f941bfe62ffffb1125031a310922dd29f774 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 13:29:38 +0200 Subject: [PATCH 066/116] Chane information filename parse AggregateParser, data_to_import all keys None value --- geotrek/common/parsers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 0ea01438ae..4fe9920494 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -932,16 +932,18 @@ def add_warning(self, key, msg): warnings.append(msg) def parse(self, filename=None, limit=None): + filename = filename if filename else self.filename if not os.path.exists(filename): - raise ImportError(_("This file doesn't exist")) + raise GlobalImportError(_(f"File does not exists at: {filename}")) with open(filename, mode='r') as f: json_aggregator = json.load(f) for key, datas in json_aggregator.items(): self.report_by_api_v2_by_type[key] = {} - if not datas.get('data_to_import'): - raise - for model in datas['data_to_import']: + models_to_import = datas.get('data_to_import') + if not models_to_import: + models_to_import = self.mapping_model_parser.keys() + for model in models_to_import: Parser = None if settings.TREKKING_TOPOLOGY_ENABLED: if model in self.invalid_model_topology: From 4d60f571a8816cdd46660e8c425b0bf0a1a8b5ab Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 13:34:11 +0200 Subject: [PATCH 067/116] Remove to_delete no kwargs GeotrekAggregator --- geotrek/common/parsers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 4fe9920494..f087250f17 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1110,10 +1110,7 @@ def start(self): } response = self.request_or_retry(self.next_url, params=params) ids = [f"{self.eid_prefix}{element['id']}" for element in response.json().get('results', [])] - if kwargs is None: - self.to_delete = set() - else: - self.to_delete = set(self.model.objects.filter(**kwargs).exclude(eid__in=ids).values_list('pk', flat=True)) + self.to_delete = set(self.model.objects.filter(**kwargs).exclude(eid__in=ids).values_list('pk', flat=True)) def filter_eid(self, src, val): return f'{self.eid_prefix}{val}' From f2ad0dbe55c8adfebbe53681ec5cb639b9fa6fb2 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 13:34:24 +0200 Subject: [PATCH 068/116] Fix params GeotrekParser --- geotrek/common/parsers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index f087250f17..f11e19fd04 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1154,12 +1154,12 @@ def next_row(self): self.model._meta.get_fields()]: updated_after = self.model.objects.filter(eid__startswith=self.eid_prefix).latest('date_update').date_update.strftime('%Y-%m-%d') - params = { - 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'portals': ','.join(portals) if portals else '', - 'updated_after': updated_after - } while self.next_url: + params = { + 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), + 'portals': ','.join(portals) if portals else '', + 'updated_after': updated_after + } response = self.request_or_retry(self.next_url, params=params) self.root = response.json() self.nb = int(self.root['count']) From eefdd3d3e913edeb7d3dde868b491651009fbfaf Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 13:35:42 +0200 Subject: [PATCH 069/116] Add test no file no_data_to_import --- .../config_aggregator_no_data_to_import.json | 22 ++++++++++ geotrek/common/tests/test_parsers.py | 44 ++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json new file mode 100644 index 0000000000..4b4195ff25 --- /dev/null +++ b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json @@ -0,0 +1,22 @@ +{ + "URL_1": { + "url": "URL_1", + "mapping": { + "trek_practice": { + "Randonnée Pédestre": [ + "Pédestre" + ], + "Vélo": [ + "VTT", + "Vélo" + ] + }, + "trek_difficulty": { + "Moyen": [ + "Facile" + ] + } + }, + "create": true + } +} \ No newline at end of file diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index c601e02151..228988271a 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -22,7 +22,7 @@ from geotrek.common.models import Organism, FileType, Attachment from geotrek.common.parsers import ( ExcelParser, AttachmentParserMixin, TourInSoftParser, ValueImportError, DownloadImportError, - TourismSystemParser, OpenSystemParser, GeotrekParser + TourismSystemParser, OpenSystemParser, GeotrekParser, GeotrekAggregatorParser ) from geotrek.common.utils.testdata import get_dummy_img @@ -483,6 +483,10 @@ class GeotrekTrekTestParser(GeotrekParser): } +class GeotrekAggregatorTestParser(GeotrekAggregatorParser): + pass + + class GeotrekParserTest(TestCase): def setUp(self, *args, **kwargs): self.filetype = FileType.objects.create(type="Photographie") @@ -491,6 +495,44 @@ def test_improperly_configurated_categories(self): with self.assertRaisesRegex(ImproperlyConfigured, 'foo_field is not configured in categories_keys_api_v2'): call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestParser', verbosity=2) + +class GeotrekAggregatorParserTest(TestCase): + def setUp(self, *args, **kwargs): + self.filetype = FileType.objects.create(type="Photographie") + + def test_geotrek_aggregator_no_file(self): + with self.assertRaisesRegex(CommandError, "File does not exists at: config_aggregator_does_not_exist.json"): + call_command('import', 'geotrek.common.tests.test_parsers.GeotrekAggregatorTestParser', + 'config_aggregator_does_not_exist.json', verbosity=0) + + @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') + @mock.patch('geotrek.common.parsers.importlib.import_module', return_value=mock.MagicMock()) + @mock.patch('django.template.loader.render_to_string') + @mock.patch('requests.get') + def test_geotrek_aggregator_no_data_to_import(self, mocked_get, mocked_render, mocked_import_module): + def mocked_json(): + return {} + + def side_effect_render(file, context): + return 'Render' + + mocked_get.json = mocked_json + mocked_get.return_value.status_code = 200 + mocked_get.return_value.content = b'' + mocked_render.side_effect = side_effect_render + output = StringIO() + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + 'config_aggregator_no_data_to_import.json') + call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2, + stdout=output) + stdout_parser = output.getvalue() + self.assertIn('Render\n', stdout_parser) + self.assertIn('0000: Trek (URL_1) (00%)', stdout_parser) + self.assertIn('0000: InformationDesk (URL_1) (00%)', stdout_parser) + self.assertIn('0000: Trek (URL_1) (00%)', stdout_parser) + # Trek, POI, Service, InformationDesk, TouristicContent, TouristicEvent, Signage, Infrastructure + self.assertEqual(8, mocked_import_module.call_count) + @skipIf(not settings.TREKKING_TOPOLOGY_ENABLED, 'Test with dynamic segmentation only') def test_geotrek_aggregator_parser_model_dynamic_segmentation(self): output = StringIO() From 63dd1b5b2e03310f681ce8019a5243eedddccd9f Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 13:40:57 +0200 Subject: [PATCH 070/116] Add tests aggregator parser --- geotrek/trekking/tests/test_parsers.py | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 42f44fdbd6..87c6f6ccbf 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -217,6 +217,78 @@ def mocked_json(): self.assertEqual(Attachment.objects.filter(object_id=trek.pk).count(), 3) self.assertEqual(Attachment.objects.get(object_id=trek.pk, license__isnull=False).license.label, "License") + @override_settings(PAPERCLIP_MAX_BYTES_SIZE_IMAGE=1) + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_create_attachment_max_size(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', + 'trek.json', 'trek_children.json', ] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'11' + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 5) + self.assertEqual(Attachment.objects.count(), 0) + + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_update_attachment(self, mocked_head, mocked_get): + + class MockResponse: + mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', + 'trek.json', 'trek_children.json', ] + mock_time = 0 + a = 0 + + def __init__(self, status_code): + self.status_code = status_code + + def json(self): + if len(self.mock_json_order) <= self.mock_time: + self.mock_time = 0 + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + @property + def content(self): + # We change content of attachment every time + self.a += 1 + return bytes(f'{self.a}', 'utf-8') + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value = MockResponse(200) + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 5) + trek = Trek.objects.all().first() + self.assertEqual(Attachment.objects.filter(object_id=trek.pk).count(), 3) + self.assertEqual(Attachment.objects.first().attachment_file.read(), b'11') + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 5) + trek.refresh_from_db() + self.assertEqual(Attachment.objects.filter(object_id=trek.pk).count(), 3) + self.assertEqual(Attachment.objects.first().attachment_file.read(), b'13') + @mock.patch('requests.get') @mock.patch('requests.head') def test_create_multiple(self, mocked_head, mocked_get): From 0add73a7a40083a8908e6ae3c9aef8de787eb169 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 15:28:36 +0200 Subject: [PATCH 071/116] Add all_datas option --- geotrek/common/parsers.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index f11e19fd04..02776e876d 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -962,7 +962,7 @@ def parse(self, filename=None, limit=None): else: Parser = parser(progress_cb=self.progress_cb, eid_prefix=key, url=datas['url'], portals_filter=datas.get('portals'), mapping=datas.get('mapping'), - create_categories=datas.get('create')) + create_categories=datas.get('create'), all_datas=datas.get('all_datas')) self.progress_cb(0, 0, f'{model} ({key})') Parser.parse() @@ -992,6 +992,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): mapping: Mapping between values in categories (example: /api/v2/touristiccontent_category/) and final values Can be use when you want to change a value from the api/v2 create_categories: Create all categories during importation + all_datas: Import all datas and do not use updated_after filter """ model = None next_url = '' @@ -1015,8 +1016,9 @@ class GeotrekParser(AttachmentParserMixin, Parser): portals_filter = None mapping = {} create_categories = False + all_datas = False - def __init__(self, create_categories=None, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): + def __init__(self, all_datas=None, create_categories=None, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): super().__init__(*args, **kwargs) self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID @@ -1025,6 +1027,7 @@ def __init__(self, create_categories=None, eid_prefix=None, mapping=None, portal self.url = url if url else self.url self.mapping = mapping if mapping else self.mapping self.eid_prefix = eid_prefix if eid_prefix else self.eid_prefix + self.all_datas = all_datas if all_datas else self.all_datas self.create_categories = create_categories if create_categories else self.create_categories self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not (f.name == 'id' or f.name == 'uuid')) self.m2m_fields = { @@ -1149,9 +1152,10 @@ def next_row(self): """ portals = self.portals_filter updated_after = None - if self.model.objects.filter(eid__startswith=self.eid_prefix).exists() and 'date_update' in [field.name for - field in - self.model._meta.get_fields()]: + + if not self.all_datas and self.model.objects.filter(eid__startswith=self.eid_prefix).exists() and 'date_update' in [field.name for + field in + self.model._meta.get_fields()]: updated_after = self.model.objects.filter(eid__startswith=self.eid_prefix).latest('date_update').date_update.strftime('%Y-%m-%d') while self.next_url: From 2fd5e17d2f6afb97d455e42eb278764ad27e3b02 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 15:29:01 +0200 Subject: [PATCH 072/116] Fix types TouristicContent Parser Aggregator --- geotrek/tourism/parsers.py | 23 +++++++++------- geotrek/tourism/tests/test_parsers.py | 39 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 28cfb4f682..8fe6a5fc96 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -943,29 +943,34 @@ def __init__(self, *args, **kwargs): self.field_options["type2"]["mapping"] = {} for r in response.json()['results']: for type_category in r['types']: - label_lang = type_category["label"][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + values = type_category["values"] id_category = type_category["id"] if self.create_categories: self.field_options['type1']["create"] = True self.field_options['type2']["create"] = True - if id_category % 10 == 1: - self.field_options['type1']["mapping"][id_category] = self.replace_mapping(label_lang, 'type1') if label_lang else None - if id_category % 10 == 2: - self.field_options['type2']["mapping"][id_category] = self.replace_mapping(label_lang, 'type2') if label_lang else None + for value in values: + if id_category % 10 == 1: + self.field_options['type1']["mapping"][value['id']] = self.replace_mapping( + value['label'][settings.MODELTRANSLATION_DEFAULT_LANGUAGE], 'type1' + ) + if id_category % 10 == 2: + self.field_options['type2']["mapping"][value['id']] = self.replace_mapping( + value['label'][settings.MODELTRANSLATION_DEFAULT_LANGUAGE], 'type2' + ) self.next_url = f"{self.url}/api/v2/touristiccontent" def filter_type1(self, src, val): type1_result = [] - for key in val.keys(): + for key, value in val.items(): if int(key) % 10 == 1: - type1_result.append(int(key)) + type1_result.extend(value) return self.apply_filter('type1', src, type1_result) def filter_type2(self, src, val): type2_result = [] - for key in val.keys(): + for key, value in val.items(): if int(key) % 10 == 2: - type2_result.append(int(key)) + type2_result.extend(value) return self.apply_filter('type2', src, type2_result) diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index c42ef0a49a..f748c688e8 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -662,6 +662,11 @@ class TestGeotrekTouristicContentParser(GeotrekTouristicContentParser): } +class TestGeotrekTouristicContentCreateCategoriesParser(GeotrekTouristicContentParser): + url = "https://test.fr" + create_categories = True + + class TestGeotrekTouristicEventParser(GeotrekTouristicEventParser): url = "https://test.fr" @@ -709,6 +714,40 @@ def mocked_json(): mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 + call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekTouristicContentCreateCategoriesParser', verbosity=0) + self.assertEqual(TouristicContent.objects.count(), 2) + touristic_content = TouristicContent.objects.all().first() + self.assertEqual(str(touristic_content.category), 'Sorties') + self.assertEqual(str(touristic_content.type1.first()), 'Ane') + self.assertEqual(str(touristic_content.name), "Balad'âne") + self.assertAlmostEqual(touristic_content.geom.x, 568112.6362873032, places=5) + self.assertAlmostEqual(touristic_content.geom.y, 6196929.676669887, places=5) + self.assertEqual(Attachment.objects.count(), 3) + + @mock.patch('requests.get') + @mock.patch('requests.head') + @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") + def test_create_create_categories(self, mocked_head, mocked_get): + self.mock_time = 0 + self.mock_json_order = ['touristiccontent_category.json', + 'touristiccontent_themes.json', + 'touristiccontent_category.json', + 'touristiccontent_ids.json', + 'touristiccontent.json'] + + def mocked_json(): + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) + + # Mock GET + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = mocked_json + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekTouristicContentParser', verbosity=0) self.assertEqual(TouristicContent.objects.count(), 2) touristic_content = TouristicContent.objects.all().first() From 9ea8418a11241bf50ba718d2d9b487acb2dc9eca Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 9 Aug 2022 17:33:58 +0200 Subject: [PATCH 073/116] Fix multiple page params was used every pages --- geotrek/common/parsers.py | 19 +++++++--- geotrek/trekking/tests/test_parsers.py | 51 ++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 02776e876d..57a64f766a 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1157,14 +1157,21 @@ def next_row(self): field in self.model._meta.get_fields()]: updated_after = self.model.objects.filter(eid__startswith=self.eid_prefix).latest('date_update').date_update.strftime('%Y-%m-%d') + params = { + 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), + 'portals': ','.join(portals) if portals else '', + 'updated_after': updated_after + } + response = self.request_or_retry(self.next_url, params=params) + self.root = response.json() + self.nb = int(self.root['count']) + + for row in self.items: + yield row + self.next_url = self.root['next'] while self.next_url: - params = { - 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'portals': ','.join(portals) if portals else '', - 'updated_after': updated_after - } - response = self.request_or_retry(self.next_url, params=params) + response = self.request_or_retry(self.next_url) self.root = response.json() self.nb = int(self.root['count']) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 87c6f6ccbf..e56ad8088a 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -217,6 +217,57 @@ def mocked_json(): self.assertEqual(Attachment.objects.filter(object_id=trek.pk).count(), 3) self.assertEqual(Attachment.objects.get(object_id=trek.pk, license__isnull=False).license.label, "License") + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_create_multiple_page(self, mocked_head, mocked_get): + class MockResponse: + mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', + 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', + 'trek.json', 'trek_children.json', ] + mock_time = 0 + a = 0 + total_mock_response = 1 + + def __init__(self, status_code): + self.status_code = status_code + + def json(self): + if len(self.mock_json_order) <= self.mock_time: + self.mock_time = 0 + self.total_mock_response += 1 + filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + data_json = json.load(f) + if self.total_mock_response == 1 and self.mock_json_order[self.mock_time] == 'test.json': + data_json['count'] = 10 + data_json['next'] = "foo" + return data_json + + @property + def content(self): + return b'' + + # Mock GET + mocked_get.return_value = MockResponse(200) + mocked_head.return_value.status_code = 200 + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2) + self.assertEqual(Trek.objects.count(), 5) + trek = Trek.objects.all().first() + self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") + self.assertEqual(trek.name_it, "Foo bar") + self.assertEqual(str(trek.difficulty), 'Très facile') + self.assertEqual(str(trek.practice), 'Cheval') + self.assertAlmostEqual(trek.geom[0][0], 569946.9850365581, places=5) + self.assertAlmostEqual(trek.geom[0][1], 6190964.893167565, places=5) + self.assertEqual(trek.children.first().name, "Foo") + self.assertEqual(trek.labels.count(), 3) + self.assertEqual(trek.labels.first().name, "Chien autorisé") + self.assertEqual(Attachment.objects.filter(object_id=trek.pk).count(), 3) + self.assertEqual(Attachment.objects.get(object_id=trek.pk, license__isnull=False).license.label, "License") + @override_settings(PAPERCLIP_MAX_BYTES_SIZE_IMAGE=1) @mock.patch('requests.get') @mock.patch('requests.head') From bee33fc16bbdb80923298535bac8e1493379d049 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Wed, 10 Aug 2022 10:38:14 +0200 Subject: [PATCH 074/116] Fix test multiple page --- geotrek/trekking/tests/test_parsers.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index e56ad8088a..a17c4d446d 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -223,26 +223,24 @@ def test_create_multiple_page(self, mocked_head, mocked_get): class MockResponse: mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', - 'trek.json', 'trek_children.json', ] + 'trek.json', 'trek_children.json', 'trek.json'] mock_time = 0 - a = 0 total_mock_response = 1 def __init__(self, status_code): self.status_code = status_code def json(self): - if len(self.mock_json_order) <= self.mock_time: - self.mock_time = 0 - self.total_mock_response += 1 filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', self.mock_json_order[self.mock_time]) - self.mock_time += 1 with open(filename, 'r') as f: data_json = json.load(f) - if self.total_mock_response == 1 and self.mock_json_order[self.mock_time] == 'test.json': + if self.mock_json_order[self.mock_time] == 'trek.json': data_json['count'] = 10 - data_json['next'] = "foo" + if self.total_mock_response == 1: + self.total_mock_response += 1 + data_json['next'] = "foo" + self.mock_time += 1 return data_json @property @@ -253,7 +251,7 @@ def content(self): mocked_get.return_value = MockResponse(200) mocked_head.return_value.status_code = 200 - call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2) + call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=0) self.assertEqual(Trek.objects.count(), 5) trek = Trek.objects.all().first() self.assertEqual(trek.name, "Boucle du Pic des Trois Seigneurs") From fb020938617c5251d1f48d37cc45a632b12a8839 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Wed, 10 Aug 2022 11:35:34 +0200 Subject: [PATCH 075/116] Add changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 617521c904..426b3f66d0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -39,6 +39,8 @@ CHANGELOG - Add possibility to use different type of file with import form - Add setting MAX_CHARACTERS for rich text fields with Mapentity 8.2.1 (#2901) - Set map resizable with Mapentity 8.2.1 (#3162) +- Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure) +- Add aggregator parser with a conductor using json file **Minor improvements** From 8356ccf2b6177fd14bc121560a0f0322e244d01d Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Wed, 10 Aug 2022 11:36:04 +0200 Subject: [PATCH 076/116] Add requests.exceptions.ConnectionError on exception attachment parser --- geotrek/common/parsers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 57a64f766a..a0b75128b6 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -604,10 +604,13 @@ def has_size_changed(self, url, attachment): if parsed_url.scheme == 'http' or parsed_url.scheme == 'https': try: response = self.request_or_retry(url, verb='head') - except DownloadImportError as e: + except (requests.exceptions.ConnectionError, DownloadImportError) as e: raise ValueImportError('Failed to load attachment: {exc}'.format(exc=e)) size = response.headers.get('content-length') - return size is not None and int(size) != attachment.attachment_file.size + try: + return size is not None and int(size) != attachment.attachment_file.size + except FileNotFoundError: + pass return True @@ -616,14 +619,14 @@ def download_attachment(self, url): if parsed_url.scheme == 'ftp': try: response = self.request_or_retry(url) - except DownloadImportError as e: + except (DownloadImportError, requests.exceptions.ConnectionError) as e: raise ValueImportError('Failed to load attachment: {exc}'.format(exc=e)) return response.read() else: if self.download_attachments: try: response = self.request_or_retry(url) - except DownloadImportError as e: + except (DownloadImportError, requests.exceptions.ConnectionError) as e: raise ValueImportError('Failed to load attachment: {exc}'.format(exc=e)) if response.status_code != requests.codes.ok: self.add_warning(_("Failed to download '{url}'").format(url=url)) From e1e23cac2ff169ade6a4b4a58c8103f3dfe910a8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Wed, 10 Aug 2022 16:51:07 +0200 Subject: [PATCH 077/116] Add tests connectionerror --- geotrek/common/tests/test_parsers.py | 40 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 228988271a..0fdbe71588 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -73,6 +73,10 @@ class AttachmentParser(AttachmentParserMixin, OrganismEidParser): non_fields = {'attachments': 'photo'} +class WarnAttachmentParser(AttachmentParser): + warn_on_missing_fields = True + + class AttachmentLegendParser(AttachmentParser): def filter_attachments(self, src, val): @@ -201,6 +205,18 @@ def test_attachment(self, mocked): self.assertEqual(attachment.filetype, self.filetype) self.assertTrue(os.path.exists(attachment.attachment_file.path), True) + @mock.patch('requests.get') + def test_attachment_connection_error(self, mocked): + mocked.return_value.status_code = 200 + mocked.side_effect = requests.exceptions.ConnectionError("Error connection") + filename = os.path.join(os.path.dirname(__file__), 'data', 'organism.xls') + output = StringIO() + output_3 = StringIO() + call_command('import', 'geotrek.common.tests.test_parsers.WarnAttachmentParser', filename, verbosity=2, + stdout=output, stderr=output_3) + self.assertFalse(Attachment.objects.exists()) + self.assertIn("Failed to load attachment: Error connection", output.getvalue()) + @mock.patch('requests.get') @override_settings(PAPERCLIP_MAX_BYTES_SIZE_IMAGE=20) def test_attachment_bigger_size(self, mocked): @@ -332,6 +348,21 @@ def test_attachment_not_updated_partially_changed(self, mocked_head, mocked_get) self.assertEqual(attachment.legend, 'legend') self.assertEqual(attachment.author, 'name') + @mock.patch('requests.get') + @mock.patch('requests.head') + def test_attachment_updated_file_not_found(self, mocked_head, mocked_get): + mocked_get.return_value.status_code = 200 + mocked_get.return_value.content = b'' + mocked_head.return_value.status_code = 200 + mocked_head.return_value.headers = {'content-length': 0} + filename = os.path.join(os.path.dirname(__file__), 'data', 'organism.xls') + call_command('import', 'geotrek.common.tests.test_parsers.AttachmentParser', filename, verbosity=0) + attachment = Attachment.objects.get() + os.remove(attachment.attachment_file.path) + call_command('import', 'geotrek.common.tests.test_parsers.AttachmentParser', filename, verbosity=0) + self.assertEqual(mocked_get.call_count, 2) + self.assertEqual(Attachment.objects.count(), 1) + @override_settings(PARSER_RETRY_SLEEP_TIME=0) @mock.patch('requests.get') @mock.patch('requests.head') @@ -363,11 +394,12 @@ def test_attachment_request_except(self, mocked_head, mocked_get): @mock.patch('geotrek.common.parsers.urlparse') def test_attachment_download_fail(self, mocked_urlparse, mocked_get): filename = os.path.join(os.path.dirname(__file__), 'data', 'organism.xls') - mocked_get.side_effect = DownloadImportError() + mocked_get.side_effect = DownloadImportError("DownloadImportError") mocked_urlparse.return_value = urllib.parse.urlparse('ftp://test.url.com/organism.xls') - - call_command('import', 'geotrek.common.tests.test_parsers.AttachmentParser', filename, verbosity=0) - + output = StringIO() + call_command('import', 'geotrek.common.tests.test_parsers.WarnAttachmentParser', filename, verbosity=2, + stdout=output) + self.assertIn("Failed to load attachment: DownloadImportError", output.getvalue()) self.assertEqual(mocked_get.call_count, 1) @mock.patch('requests.get') From 0501e4b6f5a6457ef61a2bcdee08e4b4de6b9881 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Fri, 12 Aug 2022 11:38:18 +0200 Subject: [PATCH 078/116] Improve test parser geotrek with mixin --- geotrek/common/tests/mixins.py | 12 +++ geotrek/common/tests/test_parsers.py | 14 +--- geotrek/infrastructure/tests/test_parsers.py | 14 ++-- geotrek/signage/tests/test_parsers.py | 15 ++-- geotrek/tourism/tests/test_parsers.py | 48 +++-------- geotrek/trekking/tests/test_parsers.py | 85 +++++--------------- 6 files changed, 57 insertions(+), 131 deletions(-) diff --git a/geotrek/common/tests/mixins.py b/geotrek/common/tests/mixins.py index d02b444a87..50693bfbc8 100644 --- a/geotrek/common/tests/mixins.py +++ b/geotrek/common/tests/mixins.py @@ -1,3 +1,6 @@ +import json +import os + def dictfetchall(cursor): "Return all rows from a cursor as a dict" @@ -6,3 +9,12 @@ def dictfetchall(cursor): dict(zip(columns, row)) for row in cursor.fetchall() ] + + +class GeotrekParserTestMixin: + def mock_json(self): + filename = os.path.join('geotrek', self.app_label, 'tests', 'data', 'geotrek_parser_v2', + self.mock_json_order[self.mock_time]) + self.mock_time += 1 + with open(filename, 'r') as f: + return json.load(f) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 0fdbe71588..caea49a8fc 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -3,7 +3,6 @@ from shutil import rmtree from tempfile import mkdtemp from io import StringIO -import json import requests from requests import Response import urllib @@ -20,6 +19,7 @@ from geotrek.authent.tests.factories import StructureFactory from geotrek.trekking.models import POI, Trek from geotrek.common.models import Organism, FileType, Attachment +from geotrek.common.tests.mixins import GeotrekParserTestMixin from geotrek.common.parsers import ( ExcelParser, AttachmentParserMixin, TourInSoftParser, ValueImportError, DownloadImportError, TourismSystemParser, OpenSystemParser, GeotrekParser, GeotrekAggregatorParser @@ -528,7 +528,7 @@ def test_improperly_configurated_categories(self): call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestParser', verbosity=2) -class GeotrekAggregatorParserTest(TestCase): +class GeotrekAggregatorParserTest(GeotrekParserTestMixin, TestCase): def setUp(self, *args, **kwargs): self.filetype = FileType.objects.create(type="Photographie") @@ -619,6 +619,7 @@ def test_geotrek_aggregator_parser_no_url(self): @mock.patch('requests.head') @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") def test_geotrek_aggregator_parser(self, mocked_head, mocked_get): + self.app_label = 'trekking' self.mock_time = 0 self.mock_json_order = ['trek_difficulty.json', 'trek_route.json', @@ -634,16 +635,9 @@ def test_geotrek_aggregator_parser(self, mocked_head, mocked_get): 'poi_ids.json', 'poi.json'] - def mocked_json(): - filename = os.path.join('geotrek', 'trekking', 'tests', 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 diff --git a/geotrek/infrastructure/tests/test_parsers.py b/geotrek/infrastructure/tests/test_parsers.py index 9f8413a8af..d1960b22e4 100644 --- a/geotrek/infrastructure/tests/test_parsers.py +++ b/geotrek/infrastructure/tests/test_parsers.py @@ -7,6 +7,7 @@ from django.test.utils import override_settings from geotrek.common.models import FileType +from geotrek.common.tests.mixins import GeotrekParserTestMixin from geotrek.infrastructure.models import Infrastructure from geotrek.infrastructure.parsers import GeotrekInfrastructureParser @@ -21,7 +22,9 @@ class TestGeotrekInfrastructureParser(GeotrekInfrastructureParser): } -class InfrastructureGeotrekParserTests(TestCase): +class InfrastructureGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = 'infrastructure' + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -34,16 +37,9 @@ def test_create(self, mocked_head, mocked_get): self.mock_json_order = ['infrastructure_condition.json', 'infrastructure_type.json', 'infrastructure_ids.json', 'infrastructure.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 diff --git a/geotrek/signage/tests/test_parsers.py b/geotrek/signage/tests/test_parsers.py index 0a912f94ea..75d7d3749b 100644 --- a/geotrek/signage/tests/test_parsers.py +++ b/geotrek/signage/tests/test_parsers.py @@ -7,6 +7,7 @@ from django.test.utils import override_settings from geotrek.common.models import FileType +from geotrek.common.tests.mixins import GeotrekParserTestMixin from geotrek.signage.models import Signage from geotrek.signage.parsers import GeotrekSignageParser @@ -22,7 +23,9 @@ class TestGeotrekSignageParser(GeotrekSignageParser): } -class SignageGeotrekParserTests(TestCase): +class SignageGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = "signage" + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -34,17 +37,9 @@ def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['signage_sealing.json', 'signage_condition.json', 'signage_type.json', 'signage_ids.json', 'signage.json', ] - - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index f748c688e8..57db004f20 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -9,6 +9,7 @@ from django.core.management import call_command from django.core.management.base import CommandError +from geotrek.common.tests.mixins import GeotrekParserTestMixin from geotrek.common.tests.factories import RecordSourceFactory, TargetPortalFactory from geotrek.common.models import Attachment, FileType from geotrek.common.tests import TranslationResetMixin @@ -685,7 +686,9 @@ class TestGeotrekInformationDeskParser(GeotrekInformationDeskParser): } -class TouristicContentGeotrekParserTests(TestCase): +class TouristicContentGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = "tourism" + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -701,16 +704,9 @@ def test_create(self, mocked_head, mocked_get): 'touristiccontent_ids.json', 'touristiccontent.json'] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -735,16 +731,9 @@ def test_create_create_categories(self, mocked_head, mocked_get): 'touristiccontent_ids.json', 'touristiccontent.json'] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -758,7 +747,9 @@ def mocked_json(): self.assertEqual(Attachment.objects.count(), 3) -class TouristicEventGeotrekParserTests(TestCase): +class TouristicEventGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = "tourism" + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -772,16 +763,9 @@ def test_create(self, mocked_head, mocked_get): 'touristicevent_ids.json', 'touristicevent.json'] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -794,7 +778,8 @@ def mocked_json(): self.assertAlmostEqual(touristic_event.geom.y, 6208918.713349126, places=5) -class InformationDeskGeotrekParserTests(TestCase): +class InformationDeskGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = "tourism" @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -808,13 +793,6 @@ def test_create(self, mocked_download_attachment, mocked_get): 'informationdesk.json', ] self.mock_download = 0 - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - def mocked_download(*args, **kwargs): if self.mock_download > 0: return None @@ -823,7 +801,7 @@ def mocked_download(*args, **kwargs): # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_download_attachment.side_effect = mocked_download call_command('import', 'geotrek.tourism.tests.test_parsers.TestGeotrekInformationDeskParser', verbosity=0) self.assertEqual(InformationDesk.objects.count(), 3) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index a17c4d446d..5ebd00c5da 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -11,6 +11,7 @@ from django.test.utils import override_settings from geotrek.common.models import Theme, FileType, Attachment +from geotrek.common.tests.mixins import GeotrekParserTestMixin from geotrek.trekking.models import POI, Service, Trek, DifficultyLevel, Route from geotrek.trekking.parsers import TrekParser, GeotrekPOIParser, GeotrekServiceParser, GeotrekTrekParser @@ -176,7 +177,9 @@ class TestGeotrekServiceParser(GeotrekServiceParser): @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') -class TrekGeotrekParserTests(TestCase): +class TrekGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = 'trekking' + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -189,16 +192,9 @@ def test_create(self, mocked_head, mocked_get): 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', 'trek_children.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -275,16 +271,9 @@ def test_create_attachment_max_size(self, mocked_head, mocked_get): 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', 'trek_children.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'11' mocked_head.return_value.status_code = 200 @@ -348,16 +337,9 @@ def test_create_multiple(self, mocked_head, mocked_get): 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids_2.json', 'trek_2.json', 'trek_children.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -384,16 +366,9 @@ def test_children_do_not_exist(self, mocked_head, mocked_get): 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', 'trek_children_do_not_exist.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 output = StringIO() @@ -409,16 +384,9 @@ def test_wrong_children_error(self, mocked_head, mocked_get): 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', 'trek.json', 'trek_wrong_children.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 output = StringIO() @@ -437,16 +405,9 @@ def test_updated(self, mocked_head, mocked_get): 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids_2.json', 'trek_2.json', 'trek_children.json', ] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -470,7 +431,9 @@ def mocked_json(): @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') -class POIGeotrekParserTests(TestCase): +class POIGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = "trekking" + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -482,16 +445,9 @@ def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['poi_type.json', 'poi_ids.json', 'poi.json'] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 @@ -508,7 +464,9 @@ def mocked_json(): @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') -class ServiceGeotrekParserTests(TestCase): +class ServiceGeotrekParserTests(GeotrekParserTestMixin, TestCase): + app_label = "trekking" + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") @@ -520,16 +478,9 @@ def test_create(self, mocked_head, mocked_get): self.mock_time = 0 self.mock_json_order = ['service_type.json', 'service_ids.json', 'service.json'] - def mocked_json(): - filename = os.path.join(os.path.dirname(__file__), 'data', 'geotrek_parser_v2', - self.mock_json_order[self.mock_time]) - self.mock_time += 1 - with open(filename, 'r') as f: - return json.load(f) - # Mock GET mocked_get.return_value.status_code = 200 - mocked_get.return_value.json = mocked_json + mocked_get.return_value.json = self.mock_json mocked_get.return_value.content = b'' mocked_head.return_value.status_code = 200 From 5bcbe589ec3811e1a0b92f634e57114324cc1abf Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 29 Aug 2022 14:06:02 +0200 Subject: [PATCH 079/116] Fix linting import os json classmethod --- geotrek/infrastructure/tests/test_parsers.py | 2 -- geotrek/signage/tests/test_parsers.py | 2 -- geotrek/tourism/tests/test_parsers.py | 1 + 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/geotrek/infrastructure/tests/test_parsers.py b/geotrek/infrastructure/tests/test_parsers.py index d1960b22e4..95ff064b57 100644 --- a/geotrek/infrastructure/tests/test_parsers.py +++ b/geotrek/infrastructure/tests/test_parsers.py @@ -1,6 +1,4 @@ from unittest import mock -import json -import os from django.core.management import call_command from django.test import TestCase diff --git a/geotrek/signage/tests/test_parsers.py b/geotrek/signage/tests/test_parsers.py index 75d7d3749b..35163d5b51 100644 --- a/geotrek/signage/tests/test_parsers.py +++ b/geotrek/signage/tests/test_parsers.py @@ -1,6 +1,4 @@ from unittest import mock -import json -import os from django.core.management import call_command from django.test import TestCase diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 57db004f20..50702f7f16 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -780,6 +780,7 @@ def test_create(self, mocked_head, mocked_get): class InformationDeskGeotrekParserTests(GeotrekParserTestMixin, TestCase): app_label = "tourism" + @classmethod def setUpTestData(cls): cls.filetype = FileType.objects.create(type="Photographie") From b2660472d5035945c4dd224614c51f6840085e28 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Wed, 7 Sep 2022 15:34:44 +0200 Subject: [PATCH 080/116] Add 'provider' field to Trek, POI, Service --- docs/changelog.rst | 5 ++++ docs/install/advanced-configuration.rst | 4 +++ .../trekking/locale/fr/LC_MESSAGES/django.po | 3 ++ .../migrations/0042_auto_20220907_1253.py | 28 +++++++++++++++++++ geotrek/trekking/models.py | 3 ++ .../trekking/poi_detail_attributes.html | 9 +++++- .../trekking/service_detail_attributes.html | 11 ++++++-- .../trekking/sql/post_90_defaults.sql | 4 +++ .../trekking/trek_detail_attributes.html | 6 ++++ 9 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 geotrek/trekking/migrations/0042_auto_20220907_1253.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 426b3f66d0..ceac6853d4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,11 @@ CHANGELOG 2.86.0+dev (XXXX-XX-XX) ----------------------- +**New features** + +- Add `provider` field to Trek, POI, Service (#3189) + + **Minor improvements** - Disable debug log in debian package post installation script. diff --git a/docs/install/advanced-configuration.rst b/docs/install/advanced-configuration.rst index dd1caa558b..2ec90d7ee1 100644 --- a/docs/install/advanced-configuration.rst +++ b/docs/install/advanced-configuration.rst @@ -1818,6 +1818,9 @@ A (nearly?) exhaustive list of attributes available for display and export as co "reservation_id", "portal", "uuid", + "eid", + "eid2", + "provider" ] COLUMNS_LISTS["poi_view"] = [ "structure", @@ -2289,6 +2292,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co "max_elevation", "slope", "uuid", + "provider" ] COLUMNS_LISTS["poi_export"] = [ "structure", diff --git a/geotrek/trekking/locale/fr/LC_MESSAGES/django.po b/geotrek/trekking/locale/fr/LC_MESSAGES/django.po index 3af3fe60e1..f4b21037ce 100644 --- a/geotrek/trekking/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/trekking/locale/fr/LC_MESSAGES/django.po @@ -147,6 +147,9 @@ msgstr "A bref résumé (info bulle sur la carte)" msgid "Description" msgstr "Description" +msgid "Provider" +msgstr "Fournisseur" + msgid "Complete description" msgstr "Description complète" diff --git a/geotrek/trekking/migrations/0042_auto_20220907_1253.py b/geotrek/trekking/migrations/0042_auto_20220907_1253.py new file mode 100644 index 0000000000..3edd3b315c --- /dev/null +++ b/geotrek/trekking/migrations/0042_auto_20220907_1253.py @@ -0,0 +1,28 @@ +# Generated by Django 3.1.14 on 2022-09-07 12:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('trekking', '0041_auto_20220304_1442'), + ] + + operations = [ + migrations.AddField( + model_name='poi', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + migrations.AddField( + model_name='service', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + migrations.AddField( + model_name='trek', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/trekking/models.py b/geotrek/trekking/models.py index 02bbba6c38..77e717e2a7 100755 --- a/geotrek/trekking/models.py +++ b/geotrek/trekking/models.py @@ -202,6 +202,7 @@ class Trek(Topology, StructureRelated, PicturesMixin, PublishableMixin, MapEntit verbose_name=_("Labels"), blank=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) eid2 = models.CharField(verbose_name=_("Second external id"), max_length=1024, blank=True, null=True) pois_excluded = models.ManyToManyField('Poi', related_name='excluded_treks', verbose_name=_("Excluded POIs"), blank=True) @@ -722,6 +723,7 @@ class POI(StructureRelated, PicturesMixin, PublishableMixin, MapEntityMixin, Top description = models.TextField(verbose_name=_("Description"), blank=True, help_text=_("History, details, ...")) type = models.ForeignKey('POIType', related_name='pois', verbose_name=_("Type"), on_delete=models.CASCADE) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) geometry_types_allowed = ["POINT"] @@ -853,6 +855,7 @@ class Service(StructureRelated, MapEntityMixin, Topology): on_delete=models.CASCADE) type = models.ForeignKey('ServiceType', related_name='services', verbose_name=_("Type"), on_delete=models.CASCADE) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) class Meta: verbose_name = _("Service") diff --git a/geotrek/trekking/templates/trekking/poi_detail_attributes.html b/geotrek/trekking/templates/trekking/poi_detail_attributes.html index 42226cbc2e..048af1f957 100644 --- a/geotrek/trekking/templates/trekking/poi_detail_attributes.html +++ b/geotrek/trekking/templates/trekking/poi_detail_attributes.html @@ -28,7 +28,14 @@

{% trans "Attributes" %}

{% trans "External id" %} - {{ poi.eid }} + {% if poi.eid %}{{ poi.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + {% trans "Provider" %} + {% if poi.provider %}{{ poi.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + {% include "altimetry/elevationinfo_fragment.html" %} diff --git a/geotrek/trekking/templates/trekking/service_detail_attributes.html b/geotrek/trekking/templates/trekking/service_detail_attributes.html index 3ade42bb24..fb0a448f12 100644 --- a/geotrek/trekking/templates/trekking/service_detail_attributes.html +++ b/geotrek/trekking/templates/trekking/service_detail_attributes.html @@ -15,9 +15,16 @@

{% trans "Attributes" %}

{% trans "External id" %} - {{ service.eid }} + {% if service.eid %}{{ service.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if service.provider %}{{ service.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + - {% include "altimetry/elevationinfo_fragment.html" %} {% include "mapentity/trackinfo_fragment.html" %} diff --git a/geotrek/trekking/templates/trekking/sql/post_90_defaults.sql b/geotrek/trekking/templates/trekking/sql/post_90_defaults.sql index c34f394a36..971c231b76 100644 --- a/geotrek/trekking/templates/trekking/sql/post_90_defaults.sql +++ b/geotrek/trekking/templates/trekking/sql/post_90_defaults.sql @@ -61,6 +61,7 @@ ALTER TABLE trekking_trek ALTER COLUMN accessibility_infrastructure SET DEFAULT ALTER TABLE trekking_trek ALTER COLUMN accessibility_signage SET DEFAULT ''; ALTER TABLE trekking_trek ALTER COLUMN accessibility_slope SET DEFAULT ''; ALTER TABLE trekking_trek ALTER COLUMN accessibility_width SET DEFAULT ''; +ALTER TABLE trekking_trek ALTER COLUMN provider SET DEFAULT ''; -- route -- difficulty -- web_links @@ -147,6 +148,8 @@ ALTER TABLE trekking_poi ALTER COLUMN description SET DEFAULT ''; -- name ALTER TABLE trekking_poi ALTER COLUMN review SET DEFAULT FALSE; ALTER TABLE trekking_poi ALTER COLUMN published SET DEFAULT FALSE; +ALTER TABLE trekking_poi ALTER COLUMN provider SET DEFAULT ''; + -- publication_date @@ -173,3 +176,4 @@ ALTER TABLE trekking_poi ALTER COLUMN published SET DEFAULT FALSE; -- type --eid --structure +ALTER TABLE trekking_service ALTER COLUMN provider SET DEFAULT ''; diff --git a/geotrek/trekking/templates/trekking/trek_detail_attributes.html b/geotrek/trekking/templates/trekking/trek_detail_attributes.html index da3b6817c8..56e59853f1 100644 --- a/geotrek/trekking/templates/trekking/trek_detail_attributes.html +++ b/geotrek/trekking/templates/trekking/trek_detail_attributes.html @@ -180,6 +180,12 @@

{% trans "Attributes" %}

{% else %}{% trans "None" %}{% endif %} + + {% trans "Provider" %} + {% if trek.provider %}{{ trek.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + + {% trans "Second external id" %} {% if trek.eid2 %}{{ trek.eid2|safe }} From 3703e7698a4b3434a6c22b44d223f65cb1516e33 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Wed, 7 Sep 2022 15:59:04 +0200 Subject: [PATCH 081/116] Add 'provider' field to Infrastructure, Signage --- docs/changelog.rst | 2 +- docs/install/advanced-configuration.rst | 7 +++++++ .../locale/fr/LC_MESSAGES/django.po | 3 +++ .../migrations/0031_infrastructure_provider.py | 18 ++++++++++++++++++ geotrek/infrastructure/models.py | 1 + .../infrastructure_detail_attributes.html | 12 ++++++++++++ .../infrastructure/sql/post_90_defaults.sql | 1 + .../signage/locale/fr/LC_MESSAGES/django.po | 3 +++ .../migrations/0024_signage_provider.py | 18 ++++++++++++++++++ .../signage/signage_detail_attributes.html | 12 ++++++++++++ .../templates/signage/sql/post_90_defaults.sql | 3 +++ 11 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 geotrek/infrastructure/migrations/0031_infrastructure_provider.py create mode 100644 geotrek/signage/migrations/0024_signage_provider.py diff --git a/docs/changelog.rst b/docs/changelog.rst index ceac6853d4..a8391131e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ CHANGELOG **New features** -- Add `provider` field to Trek, POI, Service (#3189) +- Add `provider` field to Trek, POI, Service, Signage, Infrastructure (#3189) **Minor improvements** diff --git a/docs/install/advanced-configuration.rst b/docs/install/advanced-configuration.rst index 2ec90d7ee1..5e8f79a827 100644 --- a/docs/install/advanced-configuration.rst +++ b/docs/install/advanced-configuration.rst @@ -1701,6 +1701,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co "min_elevation", "max_elevation", "uuid", + "provider" ] COLUMNS_LISTS["workmanagementedge_export"] = [ "eid", @@ -1726,6 +1727,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co "maintenance_difficulty", "published", "uuid", + "eid", + "provider" ] COLUMNS_LISTS["signage_view"] = [ "code", @@ -2150,6 +2153,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co "usage_difficulty", "maintenance_difficulty" "uuid", + "eid", + "provider" ] COLUMNS_LISTS["signage_export"] = [ "structure", @@ -2176,6 +2181,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co "min_elevation", "max_elevation", "uuid", + "eid", + "provider" ] COLUMNS_LISTS["intervention_export"] = [ "name", diff --git a/geotrek/infrastructure/locale/fr/LC_MESSAGES/django.po b/geotrek/infrastructure/locale/fr/LC_MESSAGES/django.po index c2030bd454..7aed0e0fcf 100644 --- a/geotrek/infrastructure/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/infrastructure/locale/fr/LC_MESSAGES/django.po @@ -117,3 +117,6 @@ msgstr "Aucun(e)" msgid "Add a new infrastructure" msgstr "Ajouter un aménagement" + +msgid "Provider" +msgstr "Fournisseur" diff --git a/geotrek/infrastructure/migrations/0031_infrastructure_provider.py b/geotrek/infrastructure/migrations/0031_infrastructure_provider.py new file mode 100644 index 0000000000..58e08fd3b3 --- /dev/null +++ b/geotrek/infrastructure/migrations/0031_infrastructure_provider.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2022-09-07 13:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('infrastructure', '0030_auto_20220314_1429'), + ] + + operations = [ + migrations.AddField( + model_name='infrastructure', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/infrastructure/models.py b/geotrek/infrastructure/models.py index 98a0ffe438..d0ee947327 100755 --- a/geotrek/infrastructure/models.py +++ b/geotrek/infrastructure/models.py @@ -99,6 +99,7 @@ class BaseInfrastructure(BasePublishableMixin, Topology, StructureRelated): on_delete=models.SET_NULL) implantation_year = models.PositiveSmallIntegerField(verbose_name=_("Implantation year"), null=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) class Meta: abstract = True diff --git a/geotrek/infrastructure/templates/infrastructure/infrastructure_detail_attributes.html b/geotrek/infrastructure/templates/infrastructure/infrastructure_detail_attributes.html index 95a3675874..dd68c02e19 100644 --- a/geotrek/infrastructure/templates/infrastructure/infrastructure_detail_attributes.html +++ b/geotrek/infrastructure/templates/infrastructure/infrastructure_detail_attributes.html @@ -44,6 +44,18 @@

{% trans "Attributes" %}

{{ object|verbose:"maintenance_difficulty" }} {{ object.maintenance_difficulty|default:"" }} + + {% trans "External id" %} + {% if object.eid %}{{ object.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if object.provider %}{{ object.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + + {% include "altimetry/elevationinfo_fragment.html" %} {% include "common/publication_info_fragment.html" %} {% include "mapentity/trackinfo_fragment.html" %} diff --git a/geotrek/infrastructure/templates/infrastructure/sql/post_90_defaults.sql b/geotrek/infrastructure/templates/infrastructure/sql/post_90_defaults.sql index f93fb42d43..c03f8a15d4 100644 --- a/geotrek/infrastructure/templates/infrastructure/sql/post_90_defaults.sql +++ b/geotrek/infrastructure/templates/infrastructure/sql/post_90_defaults.sql @@ -38,3 +38,4 @@ ALTER TABLE infrastructure_infrastructure ALTER COLUMN description SET DEFAULT ' ALTER TABLE infrastructure_infrastructure ALTER COLUMN published SET DEFAULT FALSE; -- publication_date -- structure +ALTER TABLE infrastructure_infrastructure ALTER COLUMN provider SET DEFAULT ''; \ No newline at end of file diff --git a/geotrek/signage/locale/fr/LC_MESSAGES/django.po b/geotrek/signage/locale/fr/LC_MESSAGES/django.po index bf7ee7fc92..cbe10f108e 100644 --- a/geotrek/signage/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/signage/locale/fr/LC_MESSAGES/django.po @@ -161,3 +161,6 @@ msgstr "Ajouter" msgid "meters" msgstr "mètres" + +msgid "Provider" +msgstr "Fournisseur" \ No newline at end of file diff --git a/geotrek/signage/migrations/0024_signage_provider.py b/geotrek/signage/migrations/0024_signage_provider.py new file mode 100644 index 0000000000..854d307e9f --- /dev/null +++ b/geotrek/signage/migrations/0024_signage_provider.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2022-09-07 13:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('signage', '0023_auto_20220314_1441'), + ] + + operations = [ + migrations.AddField( + model_name='signage', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/signage/templates/signage/signage_detail_attributes.html b/geotrek/signage/templates/signage/signage_detail_attributes.html index 9d795c1df1..cbc41dbc38 100644 --- a/geotrek/signage/templates/signage/signage_detail_attributes.html +++ b/geotrek/signage/templates/signage/signage_detail_attributes.html @@ -51,6 +51,18 @@ {{ object|verbose:"manager" }} {{ object.manager|default:"" }} + + {% trans "External id" %} + {% if object.eid %}{{ object.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if object.provider %}{{ object.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + + {% include "altimetry/elevationinfo_fragment.html" %} {% include "common/publication_info_fragment.html" %} {% include "mapentity/trackinfo_fragment.html" %} diff --git a/geotrek/signage/templates/signage/sql/post_90_defaults.sql b/geotrek/signage/templates/signage/sql/post_90_defaults.sql index 640bf4a3cf..93c854ab9e 100644 --- a/geotrek/signage/templates/signage/sql/post_90_defaults.sql +++ b/geotrek/signage/templates/signage/sql/post_90_defaults.sql @@ -26,6 +26,9 @@ ALTER TABLE infrastructure_infrastructure ALTER COLUMN description SET DEFAULT ' ALTER TABLE infrastructure_infrastructure ALTER COLUMN published SET DEFAULT FALSE; -- publication_date -- structure +ALTER TABLE infrastructure_infrastructure ALTER COLUMN provider SET DEFAULT ''; + + -- Direction ------------ From ca0d0b9bcb1f1efb71398597688e105c6ecf665d Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Wed, 7 Sep 2022 16:25:53 +0200 Subject: [PATCH 082/116] Add 'provider' field to TouristicContent, TouristicEvent, InformationDesk --- docs/changelog.rst | 2 +- docs/install/advanced-configuration.rst | 6 ++++ .../tourism/locale/fr/LC_MESSAGES/django.po | 3 ++ .../migrations/0026_auto_20220907_1400.py | 28 +++++++++++++++++++ geotrek/tourism/models.py | 3 ++ .../tourism/sql/post_90_defaults.sql | 3 ++ .../touristiccontent_detail_attributes.html | 10 ++++++- .../touristicevent_detail_attributes.html | 10 ++++++- 8 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 geotrek/tourism/migrations/0026_auto_20220907_1400.py diff --git a/docs/changelog.rst b/docs/changelog.rst index a8391131e3..44facc90fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ CHANGELOG **New features** -- Add `provider` field to Trek, POI, Service, Signage, Infrastructure (#3189) +- Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk (#3189) **Minor improvements** diff --git a/docs/install/advanced-configuration.rst b/docs/install/advanced-configuration.rst index 5e8f79a827..2082cc84c5 100644 --- a/docs/install/advanced-configuration.rst +++ b/docs/install/advanced-configuration.rst @@ -1881,6 +1881,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co "date_update", "date_insert", "uuid", + "eid", + "provider" ] COLUMNS_LISTS["touristic_event_view"] = [ "structure", @@ -1908,6 +1910,8 @@ A (nearly?) exhaustive list of attributes available for display and export as co "date_update", "date_insert", "uuid", + "eid", + "provider" ] COLUMNS_LISTS["feedback_view"] = [ "email", @@ -2387,6 +2391,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co "areas", "approved", "uuid", + "provider" ] COLUMNS_LISTS["touristic_event_export"] = [ "structure", @@ -2423,6 +2428,7 @@ A (nearly?) exhaustive list of attributes available for display and export as co "areas", "approved", "uuid", + "provider" ] COLUMNS_LISTS["feedback_export"] = [ "email", diff --git a/geotrek/tourism/locale/fr/LC_MESSAGES/django.po b/geotrek/tourism/locale/fr/LC_MESSAGES/django.po index 8301f831d6..eb55356569 100644 --- a/geotrek/tourism/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/tourism/locale/fr/LC_MESSAGES/django.po @@ -368,3 +368,6 @@ msgstr "Date" msgid "Invalid geometry value." msgstr "Valeur de géométrie invalide" + +msgid "Provider" +msgstr "Fournisseur" \ No newline at end of file diff --git a/geotrek/tourism/migrations/0026_auto_20220907_1400.py b/geotrek/tourism/migrations/0026_auto_20220907_1400.py new file mode 100644 index 0000000000..0b88fc9db5 --- /dev/null +++ b/geotrek/tourism/migrations/0026_auto_20220907_1400.py @@ -0,0 +1,28 @@ +# Generated by Django 3.1.14 on 2022-09-07 14:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tourism', '0025_auto_20220726_0903'), + ] + + operations = [ + migrations.AddField( + model_name='informationdesk', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + migrations.AddField( + model_name='touristiccontent', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + migrations.AddField( + model_name='touristicevent', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/tourism/models.py b/geotrek/tourism/models.py index 8efdedfb5c..32465c47ba 100644 --- a/geotrek/tourism/models.py +++ b/geotrek/tourism/models.py @@ -59,6 +59,7 @@ def __str__(self): class InformationDesk(models.Model): eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) name = models.CharField(verbose_name=_("Title"), max_length=256) type = models.ForeignKey(InformationDeskType, verbose_name=_("Type"), on_delete=models.CASCADE, related_name='desks') @@ -313,6 +314,7 @@ class TouristicContent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin on_delete=models.CASCADE, related_name='contents', blank=True, null=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) reservation_system = models.ForeignKey(ReservationSystem, verbose_name=_("Reservation system"), on_delete=models.CASCADE, blank=True, null=True) reservation_id = models.CharField(verbose_name=_("Reservation ID"), max_length=1024, @@ -426,6 +428,7 @@ class TouristicEvent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, blank=True, related_name='touristicevents', verbose_name=_("Portal")) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) approved = models.BooleanField(verbose_name=_("Approved"), default=False) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) diff --git a/geotrek/tourism/templates/tourism/sql/post_90_defaults.sql b/geotrek/tourism/templates/tourism/sql/post_90_defaults.sql index 64d3438db1..0f7b538320 100644 --- a/geotrek/tourism/templates/tourism/sql/post_90_defaults.sql +++ b/geotrek/tourism/templates/tourism/sql/post_90_defaults.sql @@ -27,6 +27,7 @@ ALTER TABLE tourism_informationdesk ALTER COLUMN accessibility SET DEFAULT ''; -- geom -- eid ALTER TABLE tourism_informationdesk ALTER COLUMN uuid SET DEFAULT gen_random_uuid(); +ALTER TABLE tourism_informationdesk ALTER COLUMN provider SET DEFAULT ''; -- TouristicContentCategory @@ -77,6 +78,7 @@ ALTER TABLE tourism_touristiccontent ALTER COLUMN published SET DEFAULT FALSE; -- structure ALTER TABLE tourism_touristiccontent ALTER COLUMN date_insert SET DEFAULT now(); ALTER TABLE tourism_touristiccontent ALTER COLUMN date_update SET DEFAULT now(); +ALTER TABLE tourism_touristiccontent ALTER COLUMN provider SET DEFAULT ''; -- deleted @@ -121,3 +123,4 @@ ALTER TABLE tourism_touristicevent ALTER COLUMN published SET DEFAULT FALSE; ALTER TABLE tourism_touristiccontent ALTER COLUMN date_insert SET DEFAULT now(); ALTER TABLE tourism_touristiccontent ALTER COLUMN date_update SET DEFAULT now(); -- deleted +ALTER TABLE tourism_touristicevent ALTER COLUMN provider SET DEFAULT ''; diff --git a/geotrek/tourism/templates/tourism/touristiccontent_detail_attributes.html b/geotrek/tourism/templates/tourism/touristiccontent_detail_attributes.html index 67b54105da..48b510e9de 100644 --- a/geotrek/tourism/templates/tourism/touristiccontent_detail_attributes.html +++ b/geotrek/tourism/templates/tourism/touristiccontent_detail_attributes.html @@ -78,7 +78,15 @@

{% trans "Attributes" %}

{% trans "External id" %} - {{ object.eid }} + {% if object.eid %}{{ object.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if object.provider %}{{ object.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + {% trans "Reservation system" %} diff --git a/geotrek/tourism/templates/tourism/touristicevent_detail_attributes.html b/geotrek/tourism/templates/tourism/touristicevent_detail_attributes.html index 8128ee25f8..4bb19bbb20 100644 --- a/geotrek/tourism/templates/tourism/touristicevent_detail_attributes.html +++ b/geotrek/tourism/templates/tourism/touristicevent_detail_attributes.html @@ -108,7 +108,15 @@

{% trans "Attributes" %}

{% trans "External id" %} - {{ object.eid }} + {% if object.eid %}{{ object.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if object.provider %}{{ object.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + {% trans "Thumbnail" %} From 351155b95af7d74298297c9e392fe0415eb5377c Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Wed, 7 Sep 2022 18:15:39 +0200 Subject: [PATCH 083/116] Enable filtering by provider --- geotrek/infrastructure/filters.py | 15 +++++++++-- geotrek/signage/filters.py | 15 +++++++++-- geotrek/tourism/filters.py | 27 ++++++++++++++++++-- geotrek/trekking/filters.py | 41 ++++++++++++++++++++++++++++--- 4 files changed, 89 insertions(+), 9 deletions(-) diff --git a/geotrek/infrastructure/filters.py b/geotrek/infrastructure/filters.py index c69e119601..5a07aaaad3 100644 --- a/geotrek/infrastructure/filters.py +++ b/geotrek/infrastructure/filters.py @@ -1,6 +1,6 @@ from django.contrib.contenttypes.models import ContentType from django.utils.translation import gettext_lazy as _ -from django_filters import CharFilter, MultipleChoiceFilter, ModelMultipleChoiceFilter +from django_filters import CharFilter, MultipleChoiceFilter, ModelMultipleChoiceFilter, ChoiceFilter from geotrek.altimetry.filters import AltimetryAllGeometriesFilterSet from geotrek.authent.filters import StructureRelatedFilterSet from geotrek.core.filters import ValidTopologyFilterSet, TopologyFilterTrail @@ -20,12 +20,23 @@ class InfrastructureFilterSet(AltimetryAllGeometriesFilterSet, ValidTopologyFilt trail = TopologyFilterTrail(label=_('Trail'), required=False) maintenance_difficulty = ModelMultipleChoiceFilter(queryset=InfrastructureMaintenanceDifficultyLevel.objects.all(), label=_("Maintenance difficulty")) usage_difficulty = ModelMultipleChoiceFilter(queryset=InfrastructureUsageDifficultyLevel.objects.all(), label=_("Usage difficulty")) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + Infrastructure.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) class Meta(StructureRelatedFilterSet.Meta): model = Infrastructure fields = StructureRelatedFilterSet.Meta.fields + [ 'category', 'type', 'condition', 'implantation_year', - 'intervention_year', 'published' + 'intervention_year', 'published', 'provider' ] def filter_intervention_year(self, qs, name, value): diff --git a/geotrek/signage/filters.py b/geotrek/signage/filters.py index 5065ffeea7..db359d06f8 100644 --- a/geotrek/signage/filters.py +++ b/geotrek/signage/filters.py @@ -1,4 +1,4 @@ -from django_filters import CharFilter, ModelChoiceFilter, MultipleChoiceFilter +from django_filters import CharFilter, ModelChoiceFilter, MultipleChoiceFilter, ChoiceFilter from django.contrib.contenttypes.models import ContentType from django.utils.translation import gettext_lazy as _ @@ -30,12 +30,23 @@ class SignageFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFi intervention_year = MultipleChoiceFilter(label=_("Intervention year"), method='filter_intervention_year', choices=Intervention.objects.year_choices()) trail = TopologyFilterTrail(label=_('Trail'), required=False) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + Signage.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) class Meta(StructureRelatedFilterSet.Meta): model = Signage fields = StructureRelatedFilterSet.Meta.fields + ['type', 'condition', 'implantation_year', 'intervention_year', 'published', 'code', 'printed_elevation', 'manager', - 'sealing'] + 'sealing', 'provider'] def filter_intervention_year(self, qs, name, value): signage_ct = ContentType.objects.get_for_model(Signage) diff --git a/geotrek/tourism/filters.py b/geotrek/tourism/filters.py index 0995e3d0bb..9cac9f2c66 100644 --- a/geotrek/tourism/filters.py +++ b/geotrek/tourism/filters.py @@ -1,6 +1,6 @@ from django.utils.translation import gettext_lazy as _ -from django_filters.filters import ModelMultipleChoiceFilter +from django_filters.filters import ModelMultipleChoiceFilter, ChoiceFilter import django_filters.rest_framework from django.db.models import Q from geotrek.authent.filters import StructureRelatedFilterSet @@ -23,12 +23,24 @@ class TypeFilter(ModelMultipleChoiceFilter): class TouristicContentFilterSet(ZoningFilterSet, StructureRelatedFilterSet): type1 = TypeFilter(queryset=TouristicContentType1.objects.all()) type2 = TypeFilter(queryset=TouristicContentType2.objects.all()) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + TouristicContent.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) class Meta(StructureRelatedFilterSet.Meta): model = TouristicContent fields = StructureRelatedFilterSet.Meta.fields + [ 'published', 'category', 'type1', 'type2', 'themes', 'approved', 'source', 'portal', 'reservation_system', + 'provider' ] @@ -72,12 +84,23 @@ class TouristicEventFilterSet(ZoningFilterSet, StructureRelatedFilterSet): after = AfterFilter(label=_("After")) before = BeforeFilter(label=_("Before")) completed = CompletedFilter(label=_("Completed")) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + TouristicEvent.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) class Meta(StructureRelatedFilterSet.Meta): model = TouristicEvent fields = StructureRelatedFilterSet.Meta.fields + [ 'published', 'type', 'themes', 'after', - 'before', 'approved', 'source', 'portal' + 'before', 'approved', 'source', 'portal', 'provider' ] diff --git a/geotrek/trekking/filters.py b/geotrek/trekking/filters.py index 3d10d261d2..3bb68a3dc8 100644 --- a/geotrek/trekking/filters.py +++ b/geotrek/trekking/filters.py @@ -1,4 +1,5 @@ from django.utils.translation import gettext_lazy as _ +from django_filters import ChoiceFilter from geotrek.authent.filters import StructureRelatedFilterSet from geotrek.core.filters import TopologyFilter, ValidTopologyFilterSet from geotrek.altimetry.filters import AltimetryPointFilterSet, AltimetryAllGeometriesFilterSet @@ -8,13 +9,24 @@ class TrekFilterSet(AltimetryAllGeometriesFilterSet, ValidTopologyFilterSet, ZoningFilterSet, StructureRelatedFilterSet): + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + Trek.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) class Meta(StructureRelatedFilterSet.Meta): model = Trek fields = StructureRelatedFilterSet.Meta.fields + [ 'published', 'difficulty', 'duration', 'themes', 'networks', 'practice', 'accessibilities', 'accessibility_level', 'route', 'labels', - 'source', 'portal', 'reservation_system', + 'source', 'portal', 'reservation_system', 'provider' ] @@ -24,15 +36,38 @@ class POITrekFilter(TopologyFilter): class POIFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFilterSet, StructureRelatedFilterSet): trek = POITrekFilter(label=_("Trek"), required=False) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + POI.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) class Meta(StructureRelatedFilterSet.Meta): model = POI fields = StructureRelatedFilterSet.Meta.fields + [ - 'published', 'type', 'trek', + 'published', 'type', 'trek', 'provider' ] class ServiceFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFilterSet, StructureRelatedFilterSet): + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=list( + map( + lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers + Service.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() + ) + ) + ) + class Meta: model = Service - fields = StructureRelatedFilterSet.Meta.fields + ['type'] + fields = StructureRelatedFilterSet.Meta.fields + ['type', 'provider'] From 530ec9c3811b73bbbd070dc933f4034ddc4b6c72 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Thu, 8 Sep 2022 12:30:23 +0200 Subject: [PATCH 084/116] Fix filtering on Services does not work --- docs/changelog.rst | 3 +++ geotrek/trekking/filters.py | 2 +- geotrek/trekking/views.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 44facc90fe..8399136fb0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,9 @@ CHANGELOG - Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk (#3189) +**Bug fixes** + +- Fix filtering on Services List does not filter **Minor improvements** diff --git a/geotrek/trekking/filters.py b/geotrek/trekking/filters.py index 3bb68a3dc8..6832e69d37 100644 --- a/geotrek/trekking/filters.py +++ b/geotrek/trekking/filters.py @@ -68,6 +68,6 @@ class ServiceFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFi ) ) - class Meta: + class Meta(StructureRelatedFilterSet.Meta): model = Service fields = StructureRelatedFilterSet.Meta.fields + ['type', 'provider'] diff --git a/geotrek/trekking/views.py b/geotrek/trekking/views.py index 0dbd14d200..c9796580a0 100755 --- a/geotrek/trekking/views.py +++ b/geotrek/trekking/views.py @@ -500,6 +500,7 @@ class ServiceViewSet(GeotrekMapentityViewSet): model = Service serializer_class = ServiceSerializer geojson_serializer_class = ServiceGeojsonSerializer + filterset_class = ServiceFilterSet def get_queryset(self): qs = self.model.objects.existing().select_related('type') From 887e5e4c6d7a2cfc324f0bfd046fa4a3f84b2cf5 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Thu, 8 Sep 2022 15:00:20 +0200 Subject: [PATCH 085/116] Fix migration errors --- geotrek/infrastructure/filters.py | 7 +----- geotrek/infrastructure/models.py | 5 ++++ geotrek/signage/filters.py | 7 +----- geotrek/signage/models.py | 5 ++++ geotrek/tourism/filters.py | 14 ++--------- .../tourism/locale/fr/LC_MESSAGES/django.po | 2 +- geotrek/tourism/models.py | 18 +++++++++++++- geotrek/trekking/filters.py | 21 +++------------- geotrek/trekking/models.py | 24 +++++++++++++++++-- 9 files changed, 57 insertions(+), 46 deletions(-) diff --git a/geotrek/infrastructure/filters.py b/geotrek/infrastructure/filters.py index 5a07aaaad3..efb8af6787 100644 --- a/geotrek/infrastructure/filters.py +++ b/geotrek/infrastructure/filters.py @@ -24,12 +24,7 @@ class InfrastructureFilterSet(AltimetryAllGeometriesFilterSet, ValidTopologyFilt field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - Infrastructure.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=Infrastructure.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): diff --git a/geotrek/infrastructure/models.py b/geotrek/infrastructure/models.py index d0ee947327..cfbf8436ef 100755 --- a/geotrek/infrastructure/models.py +++ b/geotrek/infrastructure/models.py @@ -149,6 +149,11 @@ def implantation_year_choices(self): .values_list('implantation_year', 'implantation_year') return all_years + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + class Infrastructure(MapEntityMixin, BaseInfrastructure): """ An infrastructure in the park, which is not of type SIGNAGE """ diff --git a/geotrek/signage/filters.py b/geotrek/signage/filters.py index db359d06f8..9b4bffc8f4 100644 --- a/geotrek/signage/filters.py +++ b/geotrek/signage/filters.py @@ -34,12 +34,7 @@ class SignageFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFi field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - Signage.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=Signage.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): diff --git a/geotrek/signage/models.py b/geotrek/signage/models.py index fcea619b60..ee89343ddd 100755 --- a/geotrek/signage/models.py +++ b/geotrek/signage/models.py @@ -63,6 +63,11 @@ def implantation_year_choices(self): .values_list('implantation_year', 'implantation_year') return choices + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + class Signage(MapEntityMixin, BaseInfrastructure): """ An infrastructure in the park, which is of type SIGNAGE """ diff --git a/geotrek/tourism/filters.py b/geotrek/tourism/filters.py index 9cac9f2c66..ea9d8e70d2 100644 --- a/geotrek/tourism/filters.py +++ b/geotrek/tourism/filters.py @@ -27,12 +27,7 @@ class TouristicContentFilterSet(ZoningFilterSet, StructureRelatedFilterSet): field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - TouristicContent.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=TouristicContent.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): @@ -88,12 +83,7 @@ class TouristicEventFilterSet(ZoningFilterSet, StructureRelatedFilterSet): field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - TouristicEvent.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=TouristicEvent.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): diff --git a/geotrek/tourism/locale/fr/LC_MESSAGES/django.po b/geotrek/tourism/locale/fr/LC_MESSAGES/django.po index eb55356569..1c3618b49f 100644 --- a/geotrek/tourism/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/tourism/locale/fr/LC_MESSAGES/django.po @@ -370,4 +370,4 @@ msgid "Invalid geometry value." msgstr "Valeur de géométrie invalide" msgid "Provider" -msgstr "Fournisseur" \ No newline at end of file +msgstr "Fournisseur" diff --git a/geotrek/tourism/models.py b/geotrek/tourism/models.py index 32465c47ba..0dfd2b2732 100644 --- a/geotrek/tourism/models.py +++ b/geotrek/tourism/models.py @@ -14,6 +14,7 @@ from extended_choices import Choices from geotrek.authent.models import StructureRelated +from geotrek.common.mixins.managers import NoDeleteManager from geotrek.common.mixins.models import (AddPropertyMixin, NoDeleteMixin, OptionalPictogramMixin, PictogramMixin, PicturesMixin, PublishableMixin, TimeStampedModelMixin) from geotrek.common.models import ReservationSystem, Theme @@ -275,6 +276,13 @@ class Meta: verbose_name_plural = _("Second list types") +class TouristicContentManager(NoDeleteManager): + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class TouristicContent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, MapEntityMixin, StructureRelated, TimeStampedModelMixin, PicturesMixin, NoDeleteMixin): """ A generic touristic content (accomodation, museum, etc.) in the park @@ -322,6 +330,7 @@ class TouristicContent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin approved = models.BooleanField(verbose_name=_("Approved"), default=False, help_text=_("Indicates whether the content has a label or brand")) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + objects = TouristicContentManager() class Meta: verbose_name = _("Touristic content") @@ -387,6 +396,13 @@ def __str__(self): return self.type +class TouristicEventManager(NoDeleteManager): + def provider_choices(self): + providers = self.get_queryset().existing().order_by('provider').exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class TouristicEvent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, MapEntityMixin, StructureRelated, PicturesMixin, TimeStampedModelMixin, NoDeleteMixin): """ A touristic event (conference, workshop, etc.) in the park @@ -431,7 +447,7 @@ class TouristicEvent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) approved = models.BooleanField(verbose_name=_("Approved"), default=False) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) - + objects = TouristicEventManager() id_prefix = 'E' class Meta: diff --git a/geotrek/trekking/filters.py b/geotrek/trekking/filters.py index 6832e69d37..bdf58e7d3a 100644 --- a/geotrek/trekking/filters.py +++ b/geotrek/trekking/filters.py @@ -13,12 +13,7 @@ class TrekFilterSet(AltimetryAllGeometriesFilterSet, ValidTopologyFilterSet, Zon field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - Trek.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=Trek.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): @@ -40,12 +35,7 @@ class POIFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFilter field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - POI.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=POI.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): @@ -60,12 +50,7 @@ class ServiceFilterSet(AltimetryPointFilterSet, ValidTopologyFilterSet, ZoningFi field_name='provider', empty_label=_("Provider"), label=_("Provider"), - choices=list( - map( - lambda x: (x, x), # Create list of tupples [('Provider1, 'Provider1'), ('Provider2, 'Provider2')] for distinct and not empty providers - Service.objects.exclude(provider__exact='').values_list('provider', flat=True).distinct() - ) - ) + choices=Service.objects.provider_choices() ) class Meta(StructureRelatedFilterSet.Meta): diff --git a/geotrek/trekking/models.py b/geotrek/trekking/models.py index 77e717e2a7..9b17303a24 100755 --- a/geotrek/trekking/models.py +++ b/geotrek/trekking/models.py @@ -17,7 +17,7 @@ from mapentity.serializers import plain_text from geotrek.authent.models import StructureRelated -from geotrek.core.models import Path, Topology, simplify_coords +from geotrek.core.models import Path, Topology, TopologyManager, simplify_coords from geotrek.common.utils import intersecting, classproperty from geotrek.common.mixins.models import PicturesMixin, PublishableMixin, PictogramMixin, OptionalPictogramMixin from geotrek.common.mixins.managers import NoDeleteManager @@ -111,6 +111,13 @@ class Meta: ordering = ('order', 'name') +class TrekManager(TopologyManager): + def provider_choices(self): + providers = self.get_queryset().existing().order_by('provider').distinct('provider') \ + .exclude(provider__exact='').values_list('provider', 'provider') + return providers + + class Trek(Topology, StructureRelated, PicturesMixin, PublishableMixin, MapEntityMixin): topo_object = models.OneToOneField(Topology, parent_link=True, on_delete=models.CASCADE) departure = models.CharField(verbose_name=_("Departure"), max_length=128, blank=True, @@ -215,6 +222,7 @@ class Trek(Topology, StructureRelated, PicturesMixin, PublishableMixin, MapEntit capture_map_image_waitfor = '.poi_enum_loaded.services_loaded.info_desks_loaded.ref_points_loaded' geometry_types_allowed = ["LINESTRING"] + objects = TrekManager() class Meta: verbose_name = _("Trek") @@ -718,6 +726,13 @@ def __str__(self): return "%s" % self.label +class POIManager(NoDeleteManager): + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class POI(StructureRelated, PicturesMixin, PublishableMixin, MapEntityMixin, Topology): topo_object = models.OneToOneField(Topology, parent_link=True, on_delete=models.CASCADE) description = models.TextField(verbose_name=_("Description"), blank=True, help_text=_("History, details, ...")) @@ -732,7 +747,7 @@ class Meta: verbose_name_plural = _("POI") # Override default manager - objects = NoDeleteManager() + objects = POIManager() # Do no check structure when selecting POIs to exclude check_structure_in_forms = False @@ -849,6 +864,11 @@ class ServiceManager(NoDeleteManager): def get_queryset(self): return super().get_queryset().select_related('type') + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + class Service(StructureRelated, MapEntityMixin, Topology): topo_object = models.OneToOneField(Topology, parent_link=True, From dd8e36fecc3dd33b7498e59b0d6be4288b103ec5 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Thu, 8 Sep 2022 17:44:41 +0200 Subject: [PATCH 086/116] Add 'provider' to APIv2 --- geotrek/api/tests/test_v2.py | 16 ++++++++-------- geotrek/api/v2/serializers.py | 18 +++++++++--------- geotrek/trekking/tests/test_views.py | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/geotrek/api/tests/test_v2.py b/geotrek/api/tests/test_v2.py index c34e5b781a..63aaf29a7e 100644 --- a/geotrek/api/tests/test_v2.py +++ b/geotrek/api/tests/test_v2.py @@ -66,7 +66,7 @@ 'external_id', 'gpx', 'information_desks', 'kml', 'labels', 'length_2d', 'length_3d', 'max_elevation', 'min_elevation', 'name', 'networks', 'next', 'parents', 'parking_location', 'pdf', 'points_reference', - 'portal', 'practice', 'previous', 'public_transport', 'published', 'ratings', 'ratings_description', + 'portal', 'practice', 'previous', 'public_transport', 'provider', 'published', 'ratings', 'ratings_description', 'reservation_system', 'reservation_id', 'route', 'second_external_id', 'source', 'structure', 'themes', 'update_datetime', 'url', 'uuid', 'web_links' ]) @@ -77,7 +77,7 @@ POI_PROPERTIES_GEOJSON_STRUCTURE = sorted([ 'id', 'create_datetime', 'description', 'external_id', - 'name', 'attachments', 'published', 'type', 'type_label', 'type_pictogram', + 'name', 'attachments', 'published', 'provider', 'type', 'type_label', 'type_pictogram', 'update_datetime', 'url', 'uuid' ]) @@ -92,7 +92,7 @@ TOURISTIC_CONTENT_DETAIL_JSON_STRUCTURE = sorted([ 'accessibility', 'approved', 'attachments', 'category', 'cities', 'contact', 'create_datetime', 'description', 'description_teaser', 'departure_city', 'email', 'external_id', 'geometry', 'id', 'label_accessibility', 'name', 'pdf', - 'portal', 'practical_info', 'published', 'reservation_id', 'reservation_system', + 'portal', 'practical_info', 'provider', 'published', 'reservation_id', 'reservation_system', 'source', 'structure', 'themes', 'types', 'update_datetime', 'url', 'uuid', 'website', ]) @@ -136,7 +136,7 @@ INFORMATION_DESK_PROPERTIES_JSON_STRUCTURE = sorted([ 'id', 'accessibility', 'description', 'email', 'label_accessibility', 'latitude', 'longitude', - 'municipality', 'name', 'phone', 'photo_url', 'uuid', + 'municipality', 'name', 'phone', 'photo_url', 'provider', 'uuid', 'postal_code', 'street', 'type', 'website' ]) @@ -181,7 +181,7 @@ ORGANISM_PROPERTIES_JSON_STRUCTURE = sorted(['id', 'name']) SERVICE_DETAIL_JSON_STRUCTURE = sorted([ - 'id', 'eid', 'geometry', 'structure', 'type', 'uuid' + 'id', 'eid', 'geometry', 'provider', 'structure', 'type', 'uuid' ]) SERVICE_TYPE_DETAIL_JSON_STRUCTURE = sorted([ @@ -190,7 +190,7 @@ INFRASTRUCTURE_DETAIL_JSON_STRUCTURE = sorted([ 'id', 'accessibility', 'attachments', 'condition', 'description', 'eid', 'geometry', - 'implantation_year', 'maintenance_difficulty', 'name', 'structure', + 'implantation_year', 'maintenance_difficulty', 'name', 'provider', 'structure', 'type', 'usage_difficulty', 'uuid' ]) @@ -214,7 +214,7 @@ 'id', 'accessibility', 'approved', 'attachments', 'begin_date', 'booking', 'cities', 'contact', 'create_datetime', 'description', 'description_teaser', 'duration', 'email', 'end_date', 'external_id', 'geometry', 'meeting_point', 'meeting_time', 'name', 'organizer', 'participant_number', 'pdf', 'portal', - 'practical_info', 'published', 'source', 'speaker', 'structure', 'target_audience', 'themes', + 'practical_info', 'provider', 'published', 'source', 'speaker', 'structure', 'target_audience', 'themes', 'type', 'update_datetime', 'url', 'uuid', 'website' ]) @@ -225,7 +225,7 @@ SIGNAGE_DETAIL_JSON_STRUCTURE = sorted([ 'id', 'attachments', 'blades', 'code', 'condition', 'description', 'eid', 'geometry', 'implantation_year', 'name', 'printed_elevation', 'sealing', - 'structure', 'type', 'uuid' + 'provider', 'structure', 'type', 'uuid' ]) SIGNAGE_TYPE_DETAIL_JSON_STRUCTURE = sorted([ diff --git a/geotrek/api/v2/serializers.py b/geotrek/api/v2/serializers.py index 5883ad90c7..c6cca2163f 100644 --- a/geotrek/api/v2/serializers.py +++ b/geotrek/api/v2/serializers.py @@ -181,7 +181,7 @@ class ServiceSerializer(DynamicFieldsMixin, serializers.ModelSerializer): class Meta: model = trekking_models.Service - fields = ('id', 'eid', 'geometry', 'structure', 'type', 'uuid') + fields = ('id', 'eid', 'geometry', 'provider', 'structure', 'type', 'uuid') class ReservationSystemSerializer(DynamicFieldsMixin, serializers.ModelSerializer): @@ -424,7 +424,7 @@ class Meta: 'id', 'accessibility', 'attachments', 'approved', 'category', 'description', 'description_teaser', 'departure_city', 'geometry', 'label_accessibility', 'practical_info', 'url', 'cities', 'create_datetime', - 'external_id', 'name', 'pdf', 'portal', 'published', + 'external_id', 'name', 'pdf', 'portal', 'provider', 'published', 'source', 'structure', 'themes', 'update_datetime', 'types', 'contact', 'email', 'website', 'reservation_system', 'reservation_id', 'uuid' @@ -461,7 +461,7 @@ class Meta: 'cities', 'contact', 'create_datetime', 'description', 'description_teaser', 'duration', 'email', 'end_date', 'external_id', 'geometry', 'meeting_point', 'meeting_time', 'name', 'organizer', 'participant_number', 'pdf', 'portal', - 'practical_info', 'published', 'source', 'speaker', 'structure', + 'practical_info', 'published', 'provider', 'source', 'speaker', 'structure', 'target_audience', 'themes', 'type', 'update_datetime', 'url', 'uuid', 'website' ) @@ -500,7 +500,7 @@ class Meta: fields = ( 'id', 'accessibility', 'description', 'email', 'label_accessibility', 'latitude', 'longitude', 'municipality', 'name', 'phone', 'photo_url', 'uuid', - 'postal_code', 'street', 'type', 'website' + 'postal_code', 'provider', 'street', 'type', 'website' ) @@ -699,7 +699,7 @@ class Meta: 'information_desks', 'kml', 'labels', 'length_2d', 'length_3d', 'max_elevation', 'min_elevation', 'name', 'networks', 'next', 'parents', 'parking_location', 'pdf', 'points_reference', - 'portal', 'practice', 'ratings', 'ratings_description', 'previous', 'public_transport', + 'portal', 'practice', 'provider', 'ratings', 'ratings_description', 'previous', 'public_transport', 'published', 'reservation_system', 'reservation_id', 'route', 'second_external_id', 'source', 'structure', 'themes', 'update_datetime', 'url', 'uuid', 'web_links' ) @@ -764,9 +764,9 @@ class Meta: model = trekking_models.POI fields = ( 'id', 'description', 'external_id', - 'geometry', 'name', 'attachments', 'published', 'type', + 'geometry', 'name', 'attachments', 'provider', 'published', 'type', 'type_label', 'type_pictogram', 'url', 'uuid', - 'create_datetime', 'update_datetime', + 'create_datetime', 'update_datetime' ) class ThemeSerializer(DynamicFieldsMixin, serializers.ModelSerializer): @@ -1209,7 +1209,7 @@ def get_accessibility(self, obj): class Meta: model = infrastructure_models.Infrastructure fields = ('id', 'accessibility', 'attachments', 'condition', 'description', 'eid', 'geometry', 'name', - 'implantation_year', 'maintenance_difficulty', 'structure', 'type', 'usage_difficulty', 'uuid') + 'implantation_year', 'maintenance_difficulty', 'provider', 'structure', 'type', 'usage_difficulty', 'uuid') class InfrastructureConditionSerializer(DynamicFieldsMixin, serializers.ModelSerializer): @@ -1254,7 +1254,7 @@ class SignageSerializer(DynamicFieldsMixin, serializers.ModelSerializer): class Meta: model = signage_models.Signage fields = ('id', 'attachments', 'blades', 'code', 'condition', 'description', 'eid', - 'geometry', 'implantation_year', 'name', 'printed_elevation', 'sealing', + 'geometry', 'implantation_year', 'name', 'printed_elevation', 'provider', 'sealing', 'structure', 'type', 'uuid') class SignageTypeSerializer(DynamicFieldsMixin, serializers.ModelSerializer): diff --git a/geotrek/trekking/tests/test_views.py b/geotrek/trekking/tests/test_views.py index fd59e05e8e..e7ccf60333 100755 --- a/geotrek/trekking/tests/test_views.py +++ b/geotrek/trekking/tests/test_views.py @@ -172,7 +172,7 @@ def test_listing_number_queries(self): self.modelfactory.build_batch(1000) DistrictFactory.build_batch(10) - with self.assertNumQueries(6): + with self.assertNumQueries(7): self.client.get(self.model.get_datatablelist_url()) with self.assertNumQueries(9): @@ -1479,7 +1479,7 @@ def test_listing_number_queries(self): DistrictFactory.build_batch(10) # 1) session, 2) user, 3) user perms, 4) group perms, 5) last modified, 6) list - with self.assertNumQueries(6): + with self.assertNumQueries(7): self.client.get(self.model.get_datatablelist_url()) # 1) session, 2) user, 3) user perms, 4) group perms, 5) list From bb24e93bad14bbd0eac1d925f629584bab471fc4 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 10:10:51 +0200 Subject: [PATCH 087/116] Fix tests --- geotrek/trekking/tests/test_views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geotrek/trekking/tests/test_views.py b/geotrek/trekking/tests/test_views.py index e7ccf60333..ae9792bd46 100755 --- a/geotrek/trekking/tests/test_views.py +++ b/geotrek/trekking/tests/test_views.py @@ -172,10 +172,10 @@ def test_listing_number_queries(self): self.modelfactory.build_batch(1000) DistrictFactory.build_batch(10) - with self.assertNumQueries(7): + with self.assertNumQueries(9): self.client.get(self.model.get_datatablelist_url()) - with self.assertNumQueries(9): + with self.assertNumQueries(11): self.client.get(self.model.get_format_list_url()) def test_list_in_csv(self): @@ -1483,7 +1483,7 @@ def test_listing_number_queries(self): self.client.get(self.model.get_datatablelist_url()) # 1) session, 2) user, 3) user perms, 4) group perms, 5) list - with self.assertNumQueries(5): + with self.assertNumQueries(8): self.client.get(self.model.get_format_list_url()) def test_services_on_treks_do_not_exist(self): From cf7dc9bd2d67342bb2b0997807706940c8473c26 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 11:49:16 +0200 Subject: [PATCH 088/116] Enable parsing by provider and add tests --- geotrek/common/parsers.py | 7 +++ geotrek/common/tests/test_parsers.py | 80 +++++++++++++++++++++++----- 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index a0b75128b6..80551bc6b1 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -77,6 +77,7 @@ class Parser: warn_on_missing_objects = False separator = '+' eid = None + provider = "" fields = None m2m_fields = {} constant_fields = {} @@ -265,6 +266,8 @@ def parse_obj(self, row, operation): self.add_warning(str(warnings)) return if operation == "created": + if hasattr(self.model, 'provider') and self.provider and not self.obj.provider: + self.obj.provider = self.provider self.obj.save() else: self.obj.save(update_fields=update_fields) @@ -307,6 +310,8 @@ def parse_row(self, row): self.add_warning(str(warnings)) return objects = self.model.objects.filter(**eid_kwargs) + if hasattr(self.model, 'provider'): + objects = objects.filter(provider__exact=self.provider) if len(objects) == 0 and self.update_only: if self.warn_on_missing_objects: self.add_warning(_("Bad value '{eid_val}' for field '{eid_src}'. No object with this identifier").format(eid_val=self.eid_val, eid_src=self.eid_src)) @@ -442,6 +447,8 @@ def get_to_delete_kwargs(self): kwargs[dst] = field.remote_field.model.objects.get(**filters) except field.remote_field.model.DoesNotExist: return None + if hasattr(self.model, 'provider'): + kwargs['provider__exact'] = self.provider return kwargs def start(self): diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index caea49a8fc..d52df24476 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -1,30 +1,34 @@ +import json import os -from unittest import mock, skipIf +import urllib +from io import StringIO from shutil import rmtree from tempfile import mkdtemp -from io import StringIO -import requests -from requests import Response -import urllib +from unittest import mock, skipIf -from django.test import TestCase +import requests from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.core.management.base import CommandError from django.db.utils import DatabaseError -from django.test.utils import override_settings from django.template.exceptions import TemplateDoesNotExist +from django.test import TestCase +from django.test.utils import override_settings +from requests import Response from geotrek.authent.tests.factories import StructureFactory -from geotrek.trekking.models import POI, Trek -from geotrek.common.models import Organism, FileType, Attachment +from geotrek.common.models import Attachment, FileType, Organism +from geotrek.common.parsers import (AttachmentParserMixin, DownloadImportError, + ExcelParser, GeotrekAggregatorParser, + GeotrekParser, OpenSystemParser, + TourInSoftParser, TourismSystemParser, + ValueImportError) from geotrek.common.tests.mixins import GeotrekParserTestMixin -from geotrek.common.parsers import ( - ExcelParser, AttachmentParserMixin, TourInSoftParser, ValueImportError, DownloadImportError, - TourismSystemParser, OpenSystemParser, GeotrekParser, GeotrekAggregatorParser -) from geotrek.common.utils.testdata import get_dummy_img +from geotrek.trekking.models import POI, Trek +from geotrek.trekking.parsers import GeotrekTrekParser +from geotrek.trekking.tests.factories import TrekFactory class OrganismParser(ExcelParser): @@ -515,6 +519,20 @@ class GeotrekTrekTestParser(GeotrekParser): } +class GeotrekTrekTestProviderParser(GeotrekTrekParser): + url = "https://test.fr" + provider = "Provider1" + delete = True + url_categories = {} + + +class GeotrekTrekTestNoProviderParser(GeotrekTrekParser): + url = "https://test.fr" + model = Trek + delete = True + url_categories = {} + + class GeotrekAggregatorTestParser(GeotrekAggregatorParser): pass @@ -527,6 +545,42 @@ def test_improperly_configurated_categories(self): with self.assertRaisesRegex(ImproperlyConfigured, 'foo_field is not configured in categories_keys_api_v2'): call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestParser', verbosity=2) + def mock_json(self): + filename = os.path.join('geotrek', 'common', 'tests', 'data', 'geotrek_parser_v2', 'treks.json') + with open(filename, 'r') as f: + return json.load(f) + + @mock.patch('requests.get') + def test_delete_according_to_provider(self, mocked_get): + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = self.mock_json + self.assertEqual(Trek.objects.count(), 0) + call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestProviderParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 1) + t = Trek.objects.first() + self.assertEqual(t.provider, "Provider1") + #self.assertEqual(t.uuid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") + TrekFactory(provider="Provider1", name="I should be deleted", eid="1234") + t2 = TrekFactory(provider="Provider2", name="I should not be deleted", eid="1236") + t3 = TrekFactory(provider="", name="I should not be deleted", eid="12374") + call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestProviderParser', verbosity=0) + self.assertEqual([t.pk, t2.pk, t3.pk], list(Trek.objects.values_list('pk', flat=True))) + + @mock.patch('requests.get') + def test_delete_according_to_no_provider(self, mocked_get): + mocked_get.return_value.status_code = 200 + mocked_get.return_value.json = self.mock_json + self.assertEqual(Trek.objects.count(), 0) + call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestNoProviderParser', verbosity=0) + self.assertEqual(Trek.objects.count(), 1) + t = Trek.objects.first() + self.assertEqual(t.provider, "") + #self.assertEqual(t.uuid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") + t1 = TrekFactory(provider="Provider1", name="I should not be deleted", eid="1234") + TrekFactory(provider="", name="I should be deleted", eid="12374") + call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestNoProviderParser', verbosity=0) + self.assertEqual([t.pk, t1.pk], list(Trek.objects.values_list('pk', flat=True))) + class GeotrekAggregatorParserTest(GeotrekParserTestMixin, TestCase): def setUp(self, *args, **kwargs): From 317dcbba841fe41d3b6eefb8d49c548551df0db3 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 11:55:49 +0200 Subject: [PATCH 089/116] Remove comments --- geotrek/common/tests/test_parsers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index d52df24476..d5be97d3b0 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -559,7 +559,6 @@ def test_delete_according_to_provider(self, mocked_get): self.assertEqual(Trek.objects.count(), 1) t = Trek.objects.first() self.assertEqual(t.provider, "Provider1") - #self.assertEqual(t.uuid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") TrekFactory(provider="Provider1", name="I should be deleted", eid="1234") t2 = TrekFactory(provider="Provider2", name="I should not be deleted", eid="1236") t3 = TrekFactory(provider="", name="I should not be deleted", eid="12374") @@ -575,7 +574,6 @@ def test_delete_according_to_no_provider(self, mocked_get): self.assertEqual(Trek.objects.count(), 1) t = Trek.objects.first() self.assertEqual(t.provider, "") - #self.assertEqual(t.uuid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") t1 = TrekFactory(provider="Provider1", name="I should not be deleted", eid="1234") TrekFactory(provider="", name="I should be deleted", eid="12374") call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestNoProviderParser', verbosity=0) From d48d2204dfcd157a8c0186c96d751d2e8fc2ea96 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 11:58:22 +0200 Subject: [PATCH 090/116] Codestyle --- geotrek/common/tests/test_parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index d5be97d3b0..a0b56a3d09 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -522,7 +522,7 @@ class GeotrekTrekTestParser(GeotrekParser): class GeotrekTrekTestProviderParser(GeotrekTrekParser): url = "https://test.fr" provider = "Provider1" - delete = True + delete = True url_categories = {} From 7a503fcc7489ccf44e96bdbde9a712f62279fad5 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 12:17:20 +0200 Subject: [PATCH 091/116] Add missing test data file --- .../tests/data/geotrek_parser_v2/treks.json | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 geotrek/common/tests/data/geotrek_parser_v2/treks.json diff --git a/geotrek/common/tests/data/geotrek_parser_v2/treks.json b/geotrek/common/tests/data/geotrek_parser_v2/treks.json new file mode 100644 index 0000000000..02b4e943c0 --- /dev/null +++ b/geotrek/common/tests/data/geotrek_parser_v2/treks.json @@ -0,0 +1,217 @@ +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "id": 8702, + "access": { + "fr": "Accès", + "en": "", + "es": "", + "it": "" + }, + "accessibilities": [], + "accessibility_advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_covering": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_exposure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_level": null, + "accessibility_signage": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_slope": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "accessibility_width": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advice": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "advised_parking": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "altimetric_profile": "https://foo.fr/api/v2/trek/8702/profile/", + "ambiance": { + "fr": "Ambiance", + "en": "", + "es": "", + "it": "" + }, + "arrival": { + "fr": "Étangs de Picot", + "en": "", + "es": "", + "it": "" + }, + "ascent": 797, + "attachments": [], + "attachments_accessibility": [], + "children": [], + "cities": [ + "09030" + ], + "create_datetime": "2019-04-01T13:04:06.795861Z", + "departure": { + "fr": "Barrage de Soulcem", + "en": "", + "es": "", + "it": "" + }, + "departure_city": "09030", + "departure_geom": [ + 1.4526560730280935, + 42.677959776888834 + ], + "descent": -65, + "description": { + "fr": "
Description Lorem ipsum sit dolor amet Description Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor amet

\r\n
    \r\n
  1. \r\n
    2tape numéro 1 sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDesc
    \r\n
    2. \r\n
  2. \r\n
    Prendre à droite ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lor
    \r\n
    3. \r\n
  3. Terminer tout droit
    4. \r\n
\r\n
Description Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor ametDescription Lorem ipsum sit dolor amet
", + "en": "", + "es": "", + "it": "" + }, + "description_teaser": { + "fr": "Chapeau", + "en": "", + "es": "", + "it": "" + }, + "difficulty": 3, + "disabled_infrastructure": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "duration": 5.0, + "elevation_area_url": "https://foo.fr/api/v2/trek/8702/dem/", + "elevation_svg_url": "https://foo.fr/api/v2/trek/8702/profile/?language=fr&format=svg", + "external_id": null, + "gear": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 1.4526561, + 42.6779598, + 1552.0 + ], + [ + 1.4526694, + 42.6779712, + 1554.0 + ] + ] + }, + "gpx": "https://foo.fr/api/fr/treks/8702/etangs-du-picot.gpx", + "information_desks": [], + "kml": "https://foo.fr/api/fr/treks/8702/etangs-du-picot.kml", + "labels": [], + "length_2d": 3347.5, + "length_3d": 3536.2, + "max_elevation": 2298, + "min_elevation": 1537, + "name": { + "fr": "Étangs du Picot", + "en": "", + "es": "", + "it": "" + }, + "networks": [], + "next": {}, + "parents": [], + "parking_location": null, + "pdf": { + "fr": "https://foo.fr/api/fr/treks/8702/etangs-du-picot.pdf", + "en": "https://foo.fr/api/en/treks/8702/etangs-du-picot.pdf", + "es": "https://foo.fr/api/es/treks/8702/etangs-du-picot.pdf", + "it": "https://foo.fr/api/it/treks/8702/etangs-du-picot.pdf" + }, + "points_reference": { + "type": "MultiPoint", + "coordinates": [ + [ + 1.451697349548341, + 42.6817728861541 + ], + [ + 1.456890106201172, + 42.68590558513195 + ], + [ + 1.465902328491211, + 42.68448598671003 + ] + ] + }, + "portal": [], + "practice": 4, + "ratings": [], + "ratings_description": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "previous": {}, + "public_transport": { + "fr": "", + "en": "", + "es": "", + "it": "" + }, + "published": { + "fr": true, + "en": false, + "es": false, + "it": false + }, + "reservation_system": null, + "reservation_id": "", + "route": 2, + "second_external_id": null, + "source": [], + "structure": 3, + "themes": [], + "update_datetime": "2022-04-11T14:52:42.637165Z", + "url": "https://foo.fr/api/v2/trek/8702/", + "uuid": "58ed4fc1-645d-4bf6-b956-71f0a01a5eec", + "web_links": [] + } + ] +} \ No newline at end of file From 91ab39b64c20b59a981c8cca2bb1e7b63c4476af Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 12:34:23 +0200 Subject: [PATCH 092/116] Fix tests --- geotrek/trekking/tests/test_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geotrek/trekking/tests/test_views.py b/geotrek/trekking/tests/test_views.py index ae9792bd46..6d750cc3fb 100755 --- a/geotrek/trekking/tests/test_views.py +++ b/geotrek/trekking/tests/test_views.py @@ -172,10 +172,10 @@ def test_listing_number_queries(self): self.modelfactory.build_batch(1000) DistrictFactory.build_batch(10) - with self.assertNumQueries(9): + with self.assertNumQueries(7): self.client.get(self.model.get_datatablelist_url()) - with self.assertNumQueries(11): + with self.assertNumQueries(10): self.client.get(self.model.get_format_list_url()) def test_list_in_csv(self): From 9cada5eaff59cec79d2146710882460d815b1855 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 12:50:56 +0200 Subject: [PATCH 093/116] Fix tests --- geotrek/trekking/tests/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/trekking/tests/test_views.py b/geotrek/trekking/tests/test_views.py index 6d750cc3fb..07ce885560 100755 --- a/geotrek/trekking/tests/test_views.py +++ b/geotrek/trekking/tests/test_views.py @@ -175,7 +175,7 @@ def test_listing_number_queries(self): with self.assertNumQueries(7): self.client.get(self.model.get_datatablelist_url()) - with self.assertNumQueries(10): + with self.assertNumQueries(11): self.client.get(self.model.get_format_list_url()) def test_list_in_csv(self): From a7b3ec033da4fc97e23349e00a249ba2cdf408b6 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 15:02:21 +0200 Subject: [PATCH 094/116] Set UUIDs of aggregated objects to both EID and UUID in database --- geotrek/common/parsers.py | 2 +- geotrek/common/tests/test_parsers.py | 5 ++++- geotrek/infrastructure/parsers.py | 2 +- geotrek/signage/parsers.py | 2 +- geotrek/tourism/parsers.py | 6 +++--- geotrek/trekking/parsers.py | 6 +++--- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 80551bc6b1..6ebc1cb160 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1039,7 +1039,7 @@ def __init__(self, all_datas=None, create_categories=None, eid_prefix=None, mapp self.eid_prefix = eid_prefix if eid_prefix else self.eid_prefix self.all_datas = all_datas if all_datas else self.all_datas self.create_categories = create_categories if create_categories else self.create_categories - self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not (f.name == 'id' or f.name == 'uuid')) + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') self.m2m_fields = { f.name: f.name for f in self.model._meta.many_to_many diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index a0b56a3d09..c6ff7a11d3 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -528,7 +528,6 @@ class GeotrekTrekTestProviderParser(GeotrekTrekParser): class GeotrekTrekTestNoProviderParser(GeotrekTrekParser): url = "https://test.fr" - model = Trek delete = True url_categories = {} @@ -558,6 +557,8 @@ def test_delete_according_to_provider(self, mocked_get): call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestProviderParser', verbosity=0) self.assertEqual(Trek.objects.count(), 1) t = Trek.objects.first() + self.assertEqual(t.eid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") + self.assertEqual(str(t.uuid), "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") self.assertEqual(t.provider, "Provider1") TrekFactory(provider="Provider1", name="I should be deleted", eid="1234") t2 = TrekFactory(provider="Provider2", name="I should not be deleted", eid="1236") @@ -574,6 +575,8 @@ def test_delete_according_to_no_provider(self, mocked_get): self.assertEqual(Trek.objects.count(), 1) t = Trek.objects.first() self.assertEqual(t.provider, "") + self.assertEqual(t.eid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") + self.assertEqual(str(t.uuid), "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") t1 = TrekFactory(provider="Provider1", name="I should not be deleted", eid="1234") TrekFactory(provider="", name="I should be deleted", eid="12374") call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestNoProviderParser', verbosity=0) diff --git a/geotrek/infrastructure/parsers.py b/geotrek/infrastructure/parsers.py index fd1f7b7319..c0c68c95db 100644 --- a/geotrek/infrastructure/parsers.py +++ b/geotrek/infrastructure/parsers.py @@ -12,7 +12,7 @@ class GeotrekInfrastructureParser(GeotrekParser): "deleted": False } replace_fields = { - "eid": "id", + "eid": "uuid", "geom": "geometry" } url_categories = { diff --git a/geotrek/signage/parsers.py b/geotrek/signage/parsers.py index 4e6e728480..8ecea2e08c 100644 --- a/geotrek/signage/parsers.py +++ b/geotrek/signage/parsers.py @@ -12,7 +12,7 @@ class GeotrekSignageParser(GeotrekParser): "deleted": False } replace_fields = { - "eid": "id", + "eid": "uuid", "geom": "geometry" } url_categories = { diff --git a/geotrek/tourism/parsers.py b/geotrek/tourism/parsers.py index 8fe6a5fc96..d8b4b80e08 100644 --- a/geotrek/tourism/parsers.py +++ b/geotrek/tourism/parsers.py @@ -901,7 +901,7 @@ class GeotrekTouristicContentParser(GeotrekParser): } replace_fields = { - "eid": "id", + "eid": "uuid", "geom": "geometry", } @@ -984,7 +984,7 @@ class GeotrekTouristicEventParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "id", + "eid": "uuid", "geom": "geometry" } url_categories = { @@ -1008,7 +1008,7 @@ class GeotrekInformationDeskParser(GeotrekParser): model = InformationDesk constant_fields = {} replace_fields = { - "eid": "id", + "eid": "uuid", "geom": ["latitude", "longitude"], "photo": "photo_url" } diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index d37fad3655..373e2201d6 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -74,7 +74,7 @@ class GeotrekTrekParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "id", + "eid": "uuid", "eid2": "second_external_id", "geom": "geometry" } @@ -163,7 +163,7 @@ class GeotrekServiceParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "id", + "eid": "uuid", "geom": "geometry" } url_categories = { @@ -191,7 +191,7 @@ class GeotrekPOIParser(GeotrekParser): 'deleted': False, } replace_fields = { - "eid": "id", + "eid": "uuid", "geom": "geometry" } url_categories = { From 09e310d71eed2e2e1ab5f63458523abc0e5c1d92 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 15:04:01 +0200 Subject: [PATCH 095/116] Fix changelog --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8399136fb0..10962c671b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,8 @@ CHANGELOG **New features** - Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk (#3189) +- Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure) +- Add aggregator parser with a conductor using json file **Bug fixes** @@ -47,8 +49,6 @@ CHANGELOG - Add possibility to use different type of file with import form - Add setting MAX_CHARACTERS for rich text fields with Mapentity 8.2.1 (#2901) - Set map resizable with Mapentity 8.2.1 (#3162) -- Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure) -- Add aggregator parser with a conductor using json file **Minor improvements** From cff574f0ac0a6fb61cb6890c0a712499a7153da6 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 15:26:45 +0200 Subject: [PATCH 096/116] Add 'provider' field to Path, Trail --- docs/changelog.rst | 3 ++- geotrek/api/tests/test_v2.py | 2 +- geotrek/api/v2/serializers.py | 2 +- geotrek/core/locale/fr/LC_MESSAGES/django.po | 3 +++ geotrek/core/models.py | 2 ++ .../templates/core/path_detail_attributes.html | 15 ++++++++++++++- .../core/templates/core/sql/post_90_defaults.sql | 4 ++++ .../templates/core/trail_detail_attributes.html | 12 ++++++++++++ geotrek/outdoor/locale/fr/LC_MESSAGES/django.po | 3 +++ .../sensitivity/locale/fr/LC_MESSAGES/django.po | 3 +++ 10 files changed, 45 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 10962c671b..d1897f2bdb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,8 @@ CHANGELOG **New features** -- Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk (#3189) +- Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk, + Path, Trail (#3189) - Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure) - Add aggregator parser with a conductor using json file diff --git a/geotrek/api/tests/test_v2.py b/geotrek/api/tests/test_v2.py index 63aaf29a7e..6943bbfbcc 100644 --- a/geotrek/api/tests/test_v2.py +++ b/geotrek/api/tests/test_v2.py @@ -71,7 +71,7 @@ 'themes', 'update_datetime', 'url', 'uuid', 'web_links' ]) -PATH_PROPERTIES_GEOJSON_STRUCTURE = sorted(['comments', 'length_2d', 'length_3d', 'name', 'url', 'uuid']) +PATH_PROPERTIES_GEOJSON_STRUCTURE = sorted(['comments', 'length_2d', 'length_3d', 'name', 'provider', 'url', 'uuid']) TOUR_PROPERTIES_GEOJSON_STRUCTURE = sorted(TREK_PROPERTIES_GEOJSON_STRUCTURE + ['count_children', 'steps']) diff --git a/geotrek/api/v2/serializers.py b/geotrek/api/v2/serializers.py index c6cca2163f..98e4365859 100644 --- a/geotrek/api/v2/serializers.py +++ b/geotrek/api/v2/serializers.py @@ -518,7 +518,7 @@ class Meta: model = core_models.Path fields = ( 'id', 'comments', 'geometry', 'length_2d', 'length_3d', - 'name', 'url', 'uuid' + 'name', 'provider', 'url', 'uuid' ) diff --git a/geotrek/core/locale/fr/LC_MESSAGES/django.po b/geotrek/core/locale/fr/LC_MESSAGES/django.po index 112ac5df3a..04d7fc96eb 100644 --- a/geotrek/core/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/core/locale/fr/LC_MESSAGES/django.po @@ -177,6 +177,9 @@ msgstr "Interventions" msgid "Offset" msgstr "Décalage" +msgid "Provider" +msgstr "Fournisseur" + msgid "Kind" msgstr "Type de topologie" diff --git a/geotrek/core/models.py b/geotrek/core/models.py index a00c468070..7794c42564 100644 --- a/geotrek/core/models.py +++ b/geotrek/core/models.py @@ -91,6 +91,7 @@ class Path(ZoningPropertiesMixin, AddPropertyMixin, MapEntityMixin, AltimetryMix blank=True, related_name="paths", verbose_name=_("Networks")) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) draft = models.BooleanField(default=False, verbose_name=_("Draft"), db_index=True) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) @@ -970,6 +971,7 @@ class Trail(MapEntityMixin, Topology, StructureRelated): arrival = models.CharField(verbose_name=_("Arrival"), blank=True, max_length=64) comments = models.TextField(default="", blank=True, verbose_name=_("Comments")) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) certifications_verbose_name = _("Certifications") geometry_types_allowed = ["LINESTRING"] diff --git a/geotrek/core/templates/core/path_detail_attributes.html b/geotrek/core/templates/core/path_detail_attributes.html index 69262fa51e..6736e81e65 100644 --- a/geotrek/core/templates/core/path_detail_attributes.html +++ b/geotrek/core/templates/core/path_detail_attributes.html @@ -45,7 +45,20 @@

{% trans "Attributes" %}

{% if path.source %}{{ path.source }} {% else %}{% trans "None" %}{% endif %} - + + + {% trans "External id" %} + {% if path.eid %}{{ path.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if path.provider %}{{ path.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + + + {{ path|verbose:"stake" }} {% if path.stake %}{{ path.stake }} {% else %}{% trans "None" %}{% endif %} diff --git a/geotrek/core/templates/core/sql/post_90_defaults.sql b/geotrek/core/templates/core/sql/post_90_defaults.sql index 58602d272f..2a3da9be7f 100644 --- a/geotrek/core/templates/core/sql/post_90_defaults.sql +++ b/geotrek/core/templates/core/sql/post_90_defaults.sql @@ -25,6 +25,8 @@ ALTER TABLE core_path ALTER COLUMN max_elevation SET DEFAULT 0; ALTER TABLE core_path ALTER COLUMN slope SET DEFAULT 0.0; ALTER TABLE core_path ALTER COLUMN date_insert SET DEFAULT now(); ALTER TABLE core_path ALTER COLUMN date_update SET DEFAULT now(); +ALTER TABLE core_path ALTER COLUMN provider SET DEFAULT ''; + -- structure @@ -91,4 +93,6 @@ ALTER TABLE core_trail ALTER COLUMN departure SET DEFAULT ''; ALTER TABLE core_trail ALTER COLUMN arrival SET DEFAULT ''; ALTER TABLE core_trail ALTER COLUMN comments SET DEFAULT ''; ALTER TABLE core_trail ALTER COLUMN eid SET DEFAULT ''; +ALTER TABLE core_trail ALTER COLUMN provider SET DEFAULT ''; + -- structure diff --git a/geotrek/core/templates/core/trail_detail_attributes.html b/geotrek/core/templates/core/trail_detail_attributes.html index d534bb9dc5..2b05d58d28 100644 --- a/geotrek/core/templates/core/trail_detail_attributes.html +++ b/geotrek/core/templates/core/trail_detail_attributes.html @@ -36,6 +36,18 @@

{% trans "Attributes" %}

{% include "altimetry/elevationinfo_fragment.html" %} {% include "mapentity/trackinfo_fragment.html" %} + + {% trans "External id" %} + {% if trail.eid %}{{ trail.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if trail.provider %}{{ trail.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + + {{ block.super }} diff --git a/geotrek/outdoor/locale/fr/LC_MESSAGES/django.po b/geotrek/outdoor/locale/fr/LC_MESSAGES/django.po index 781c815511..fa8460bbef 100644 --- a/geotrek/outdoor/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/outdoor/locale/fr/LC_MESSAGES/django.po @@ -23,6 +23,9 @@ msgstr "Couleur" msgid "Outdoor" msgstr "Outdoor" +msgid "Provider" +msgstr "Fournisseur" + msgid "Sector" msgstr "Filière" diff --git a/geotrek/sensitivity/locale/fr/LC_MESSAGES/django.po b/geotrek/sensitivity/locale/fr/LC_MESSAGES/django.po index b0f97fea2b..0992ea36ed 100644 --- a/geotrek/sensitivity/locale/fr/LC_MESSAGES/django.po +++ b/geotrek/sensitivity/locale/fr/LC_MESSAGES/django.po @@ -71,6 +71,9 @@ msgstr "Novembre" msgid "Decembre" msgstr "Décembre" +msgid "Provider" +msgstr "Fournisseur" + msgid "Sport practices" msgstr "Pratiques sportives" From e3eee3e44174580b73560206daa36a7eabbdd100 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 15:36:14 +0200 Subject: [PATCH 097/116] Add 'provider' field to Site, Course --- docs/changelog.rst | 2 +- geotrek/api/tests/test_v2.py | 4 ++-- geotrek/api/v2/serializers.py | 4 ++-- .../migrations/0034_auto_20220909_1316.py | 23 +++++++++++++++++++ .../migrations/0040_auto_20220909_1327.py | 23 +++++++++++++++++++ geotrek/outdoor/models.py | 2 ++ .../outdoor/course_detail_attributes.html | 10 +++++++- .../outdoor/site_detail_attributes.html | 10 +++++++- .../outdoor/sql/post_90_defaults.sql | 3 ++- 9 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 geotrek/core/migrations/0034_auto_20220909_1316.py create mode 100644 geotrek/outdoor/migrations/0040_auto_20220909_1327.py diff --git a/docs/changelog.rst b/docs/changelog.rst index d1897f2bdb..74a95636f4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ CHANGELOG **New features** - Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk, - Path, Trail (#3189) + Path, Trail, Course, Site (#3189) - Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure) - Add aggregator parser with a conductor using json file diff --git a/geotrek/api/tests/test_v2.py b/geotrek/api/tests/test_v2.py index 6943bbfbcc..f546741265 100644 --- a/geotrek/api/tests/test_v2.py +++ b/geotrek/api/tests/test_v2.py @@ -147,7 +147,7 @@ SITE_PROPERTIES_JSON_STRUCTURE = sorted([ 'accessibility', 'advice', 'ambiance', 'attachments', 'children', 'cities', 'courses', 'description', 'description_teaser', 'eid', 'geometry', 'id', 'information_desks', 'labels', 'managers', 'name', 'orientation', 'parent', 'period', 'portal', - 'practice', 'pdf', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', 'wind', 'web_links', + 'practice', 'provider', 'pdf', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', 'wind', 'web_links', ]) OUTDOORPRACTICE_PROPERTIES_JSON_STRUCTURE = sorted(['id', 'name', 'sector', 'pictogram']) @@ -172,7 +172,7 @@ COURSE_PROPERTIES_JSON_STRUCTURE = sorted([ 'accessibility', 'advice', 'cities', 'description', 'eid', 'equipment', 'geometry', 'height', 'id', 'length', 'name', 'ratings', 'ratings_description', 'sites', 'structure', - 'type', 'url', 'attachments', 'max_elevation', 'min_elevation', 'parents', + 'type', 'url', 'attachments', 'max_elevation', 'min_elevation', 'parents', 'provider', 'pdf', 'points_reference', 'children', 'duration', 'gear', 'uuid' ]) diff --git a/geotrek/api/v2/serializers.py b/geotrek/api/v2/serializers.py index 98e4365859..55323d81dd 100644 --- a/geotrek/api/v2/serializers.py +++ b/geotrek/api/v2/serializers.py @@ -1055,7 +1055,7 @@ class Meta: fields = ( 'id', 'accessibility', 'advice', 'ambiance', 'attachments', 'cities', 'children', 'description', 'description_teaser', 'eid', 'geometry', 'information_desks', 'labels', 'managers', - 'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice', + 'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice', 'provider' 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', 'courses', 'web_links', 'wind', ) @@ -1115,7 +1115,7 @@ class Meta: fields = ( 'id', 'accessibility', 'advice', 'attachments', 'children', 'cities', 'description', 'duration', 'eid', 'equipment', 'gear', 'geometry', 'height', 'length', 'max_elevation', - 'min_elevation', 'name', 'parents', 'pdf', 'points_reference', 'ratings', 'ratings_description', + 'min_elevation', 'name', 'parents', 'pdf', 'points_reference', 'provider', 'ratings', 'ratings_description', 'sites', 'structure', 'type', 'url', 'uuid' ) diff --git a/geotrek/core/migrations/0034_auto_20220909_1316.py b/geotrek/core/migrations/0034_auto_20220909_1316.py new file mode 100644 index 0000000000..b7db1254fa --- /dev/null +++ b/geotrek/core/migrations/0034_auto_20220909_1316.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.14 on 2022-09-09 13:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0033_auto_20220728_0940'), + ] + + operations = [ + migrations.AddField( + model_name='path', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + migrations.AddField( + model_name='trail', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/outdoor/migrations/0040_auto_20220909_1327.py b/geotrek/outdoor/migrations/0040_auto_20220909_1327.py new file mode 100644 index 0000000000..e37f9b94d1 --- /dev/null +++ b/geotrek/outdoor/migrations/0040_auto_20220909_1327.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.14 on 2022-09-09 13:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('outdoor', '0039_auto_20220304_1442'), + ] + + operations = [ + migrations.AddField( + model_name='course', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + migrations.AddField( + model_name='site', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index 8ab6c4d536..339ab8caf1 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -171,6 +171,7 @@ class Site(ZoningPropertiesMixin, AddPropertyMixin, PicturesMixin, PublishableMi type = models.ForeignKey(SiteType, related_name="sites", on_delete=models.PROTECT, verbose_name=_("Type"), null=True, blank=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) managers = models.ManyToManyField(Organism, verbose_name=_("Managers"), blank=True) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) @@ -355,6 +356,7 @@ class Course(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, MapEntit ratings = models.ManyToManyField(Rating, related_name='courses', blank=True, verbose_name=_("Ratings")) height = models.IntegerField(verbose_name=_("Height"), blank=True, null=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) type = models.ForeignKey(CourseType, related_name="courses", on_delete=models.PROTECT, verbose_name=_("Type"), null=True, blank=True) pois_excluded = models.ManyToManyField('trekking.Poi', related_name='excluded_courses', verbose_name=_("Excluded POIs"), diff --git a/geotrek/outdoor/templates/outdoor/course_detail_attributes.html b/geotrek/outdoor/templates/outdoor/course_detail_attributes.html index 1bdf7c13a3..092f126711 100644 --- a/geotrek/outdoor/templates/outdoor/course_detail_attributes.html +++ b/geotrek/outdoor/templates/outdoor/course_detail_attributes.html @@ -87,7 +87,15 @@

{% trans "Attributes" %}

{% trans "External id" %} - {% if object.eid %}{{ object.eid }}{% else %}{% trans "None" %}{% endif %} + {% if object.eid %}{{ object.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if object.provider %}{{ object.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + {% include "common/publication_info_fragment.html" %} {% include "altimetry/elevationinfo_fragment.html" %} diff --git a/geotrek/outdoor/templates/outdoor/site_detail_attributes.html b/geotrek/outdoor/templates/outdoor/site_detail_attributes.html index 71a55204b5..23c97bc687 100644 --- a/geotrek/outdoor/templates/outdoor/site_detail_attributes.html +++ b/geotrek/outdoor/templates/outdoor/site_detail_attributes.html @@ -159,7 +159,15 @@

{% trans "Attributes" %}

{% trans "External id" %} - {% if object.eid %}{{ object.eid }}{% else %}{% trans "None" %}{% endif %} + {% if object.eid %}{{ object.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if object.provider %}{{ object.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + {% include "common/publication_info_fragment.html" %} {% include "altimetry/elevationinfo_fragment.html" %} diff --git a/geotrek/outdoor/templates/outdoor/sql/post_90_defaults.sql b/geotrek/outdoor/templates/outdoor/sql/post_90_defaults.sql index 61d55e3329..b678ee4972 100644 --- a/geotrek/outdoor/templates/outdoor/sql/post_90_defaults.sql +++ b/geotrek/outdoor/templates/outdoor/sql/post_90_defaults.sql @@ -74,7 +74,7 @@ ALTER TABLE outdoor_site ALTER COLUMN accessibility SET DEFAULT ''; -- eid -- managers ALTER TABLE outdoor_site ALTER COLUMN uuid SET DEFAULT gen_random_uuid(); - +ALTER TABLE outdoor_site ALTER COLUMN provider SET DEFAULT ''; -- OrderedCourseChild --------------------- @@ -115,3 +115,4 @@ ALTER TABLE outdoor_course ALTER COLUMN max_elevation SET DEFAULT 0; ALTER TABLE outdoor_course ALTER COLUMN slope SET DEFAULT 0.0; ALTER TABLE outdoor_course ALTER COLUMN date_insert SET DEFAULT now(); ALTER TABLE outdoor_course ALTER COLUMN date_update SET DEFAULT now(); +ALTER TABLE outdoor_course ALTER COLUMN provider SET DEFAULT ''; From 785d7d89aa2e38fdbca5d243645eaa840ecbbe0e Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 15:57:07 +0200 Subject: [PATCH 098/116] Add 'provider' field to SensitiveArea --- docs/changelog.rst | 2 +- geotrek/api/tests/test_v2.py | 2 +- .../migrations/0022_sensitivearea_provider.py | 18 ++++++++++++++++++ geotrek/sensitivity/models.py | 1 + .../sensitivearea_detail_attributes.html | 12 ++++++++++++ .../sensitivity/sql/post_90_defaults.sql | 1 + 6 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 geotrek/sensitivity/migrations/0022_sensitivearea_provider.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 74a95636f4..06104921c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ CHANGELOG **New features** - Add `provider` field to Trek, POI, Service, Signage, Infrastructure, TouristicContent, TouristicEvent, InformationDesk, - Path, Trail, Course, Site (#3189) + Path, Trail, Course, Site, SensitiveArea (#3189) - Add parser using api v2 (InformationDesk, TouristicContent, TouristicEvent, POI, Trek, Service, Signage, Infrastructure) - Add aggregator parser with a conductor using json file diff --git a/geotrek/api/tests/test_v2.py b/geotrek/api/tests/test_v2.py index f546741265..ee0cecbffc 100644 --- a/geotrek/api/tests/test_v2.py +++ b/geotrek/api/tests/test_v2.py @@ -158,7 +158,7 @@ SENSITIVE_AREA_PROPERTIES_JSON_STRUCTURE = sorted([ 'id', 'contact', 'create_datetime', 'description', 'elevation', 'geometry', - 'info_url', 'kml_url', 'name', 'period', 'practices', 'published', 'species_id', + 'info_url', 'kml_url', 'name', 'period', 'practices', 'provider', 'published', 'species_id', 'structure', 'update_datetime', 'url' ]) diff --git a/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py b/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py new file mode 100644 index 0000000000..53c8da720b --- /dev/null +++ b/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2022-09-09 13:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sensitivity', '0021_auto_20210218_0734'), + ] + + operations = [ + migrations.AddField( + model_name='sensitivearea', + name='provider', + field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + ), + ] diff --git a/geotrek/sensitivity/models.py b/geotrek/sensitivity/models.py index b21723c3cb..03eb19f7c1 100644 --- a/geotrek/sensitivity/models.py +++ b/geotrek/sensitivity/models.py @@ -81,6 +81,7 @@ class SensitiveArea(MapEntityMixin, StructureRelated, TimeStampedModelMixin, NoD description = models.TextField(verbose_name=_("Description"), blank=True) contact = models.TextField(verbose_name=_("Contact"), blank=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) + provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) class Meta: verbose_name = _("Sensitive area") diff --git a/geotrek/sensitivity/templates/sensitivity/sensitivearea_detail_attributes.html b/geotrek/sensitivity/templates/sensitivity/sensitivearea_detail_attributes.html index 98993b7b47..b27c853ce3 100644 --- a/geotrek/sensitivity/templates/sensitivity/sensitivearea_detail_attributes.html +++ b/geotrek/sensitivity/templates/sensitivity/sensitivearea_detail_attributes.html @@ -49,6 +49,18 @@

{% trans "Attributes" %}

{% include "common/publication_info_fragment.html" %} {% include "mapentity/trackinfo_fragment.html" %} + + {% trans "External id" %} + {% if sensitivearea.eid %}{{ sensitivearea.eid|safe }} + {% else %}{% trans "None" %}{% endif %} + + + + {% trans "Provider" %} + {% if sensitivearea.provider %}{{ sensitivearea.provider|safe }} + {% else %}{% trans "None" %}{% endif %} + + {{ block.super }} diff --git a/geotrek/sensitivity/templates/sensitivity/sql/post_90_defaults.sql b/geotrek/sensitivity/templates/sensitivity/sql/post_90_defaults.sql index de9bd22dba..14d93a40a7 100644 --- a/geotrek/sensitivity/templates/sensitivity/sql/post_90_defaults.sql +++ b/geotrek/sensitivity/templates/sensitivity/sql/post_90_defaults.sql @@ -38,4 +38,5 @@ ALTER TABLE sensitivity_sensitivearea ALTER COLUMN contact SET DEFAULT ''; -- structure ALTER TABLE maintenance_project ALTER COLUMN date_insert SET DEFAULT now(); ALTER TABLE maintenance_project ALTER COLUMN date_update SET DEFAULT now(); +ALTER TABLE sensitivity_sensitivearea ALTER COLUMN provider SET DEFAULT ''; -- deleted From 9dd24f2158f62494595fb443cb45cbe2b1c6a4c9 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 16:17:02 +0200 Subject: [PATCH 099/116] Fix typo in serializer --- geotrek/api/v2/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geotrek/api/v2/serializers.py b/geotrek/api/v2/serializers.py index 55323d81dd..c5e53adb00 100644 --- a/geotrek/api/v2/serializers.py +++ b/geotrek/api/v2/serializers.py @@ -845,7 +845,7 @@ class Meta: fields = ( 'id', 'contact', 'create_datetime', 'description', 'elevation', 'geometry', 'info_url', 'kml_url', 'name', 'period', - 'practices', 'published', 'species_id', 'structure', + 'practices', 'published', 'species_id', 'provider', 'structure', 'update_datetime', 'url' ) @@ -1055,7 +1055,7 @@ class Meta: fields = ( 'id', 'accessibility', 'advice', 'ambiance', 'attachments', 'cities', 'children', 'description', 'description_teaser', 'eid', 'geometry', 'information_desks', 'labels', 'managers', - 'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice', 'provider' + 'name', 'orientation', 'pdf', 'period', 'parent', 'portal', 'practice', 'provider', 'ratings', 'sector', 'source', 'structure', 'themes', 'type', 'url', 'uuid', 'courses', 'web_links', 'wind', ) From a3b725a5c1ee38d139ab8693d365836139b2842a Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 16:51:56 +0200 Subject: [PATCH 100/116] Fix use uuid as id in parsers --- geotrek/common/parsers.py | 3 ++- .../tourism/tests/data/geotrek_parser_v2/informationdesk.json | 3 +++ .../tests/data/geotrek_parser_v2/informationdesk_ids.json | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 6ebc1cb160..2b5f7aea76 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1121,8 +1121,9 @@ def start(self): 'fields': 'id', 'page_size': 10000 } + json_key = self.replace_fields.get('eid', 'id') response = self.request_or_retry(self.next_url, params=params) - ids = [f"{self.eid_prefix}{element['id']}" for element in response.json().get('results', [])] + ids = [f"{self.eid_prefix}{element[json_key]}" for element in response.json().get('results', [])] self.to_delete = set(self.model.objects.filter(**kwargs).exclude(eid__in=ids).values_list('pk', flat=True)) def filter_eid(self, src, val): diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json index 95c149b433..c4c40ee382 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk.json @@ -30,6 +30,7 @@ }, "phone": "09 64 46 55 12", "photo_url": "https://foo.fr/media/upload/office-seix.jpg.150x150_q85.jpg", + "uuid": "576198ee-b407-4e3c-9f3f-0b5efef02016", "postal_code": "09140", "street": "33 Place Champ de Mars", "type": { @@ -71,6 +72,7 @@ }, "phone": null, "photo_url": "https://foo.fr/media/upload/office-seix.jpg.150x150_q85.jpg", + "uuid": "576198ee-b407-4e3c-9f3f-0b5efef02014", "postal_code": "31300", "street": "52 rue Jacques Babinet", "type": { @@ -112,6 +114,7 @@ }, "phone": null, "photo_url": "", + "uuid": "576198ee-b407-4e3c-9f3f-0b5efef02015", "postal_code": null, "street": null, "type": { diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json index 20ea9e4cef..e113fcd438 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/informationdesk_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 1 + "uuid": "576198ee-b407-4e3c-9f3f-0b5efef02016" }, { - "id": 2 + "uuid": "576198ee-b407-4e3c-9f3f-0b5efef02014" } ] } \ No newline at end of file From 07b2cae0a9a2e06e6b7fed8304af5a8b955d7742 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 9 Sep 2022 17:18:26 +0200 Subject: [PATCH 101/116] Fix use uuid as id in parsers tests --- geotrek/sensitivity/tests/test_views.py | 1 + .../trekking/tests/data/geotrek_parser_v2/poi_ids.json | 4 ++-- .../tests/data/geotrek_parser_v2/service_ids.json | 4 ++-- .../tests/data/geotrek_parser_v2/trek_ids.json | 10 +++++----- .../tests/data/geotrek_parser_v2/trek_ids_2.json | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/geotrek/sensitivity/tests/test_views.py b/geotrek/sensitivity/tests/test_views.py index 7fafa09971..5cd071e2ba 100644 --- a/geotrek/sensitivity/tests/test_views.py +++ b/geotrek/sensitivity/tests/test_views.py @@ -96,6 +96,7 @@ def setUp(self): "name": self.species.name, "period": [False, False, False, False, False, True, True, False, False, False, False, False], 'practices': [p.pk for p in self.species.practices.all()], + 'provider': '', 'structure': 'My structure', 'published': True, } diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json index 962735c479..470f590d2f 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 2867 + "uuid": "fd30f548-d870-402b-bbaf-31f895416662" }, { - "id": 2869 + "uuid": "542afd02-487a-4df7-afaf-2082b24e8b6e" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json index 47cfe79b80..9183cf8824 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 4886 + "uuid": "1c67978b-9833-4f31-a384-c76a05eb2fbf" }, { - "id": 10393 + "uuid": "d2acb966-d01c-4eb1-81f9-a9f490c8484a" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json index 0c78b12976..62c9444a8e 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json @@ -4,19 +4,19 @@ "previous": null, "results": [ { - "id": 2 + "suuid": "9e70b294-1134-4c50-9c56-d722720cacf1" }, { - "id": 10443 + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12" }, { - "id": 2849 + "uuid": "6761143f-9244-41d0-b1af-21114408f769" }, { - "id": 10439 + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" }, { - "id": 8700 + "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json index 1fdcf2dd15..8541f394b4 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 8702 + "uuid": "58ed4fc1-645d-4bf6-b956-71f0a01a5eec" }, { - "id": 2 + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1" } ] } \ No newline at end of file From e64c383001cac721f6cc263d6db64f325c77a0c7 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Mon, 12 Sep 2022 16:01:16 +0200 Subject: [PATCH 102/116] Fix use uuid as id in Trek parsers --- geotrek/common/parsers.py | 6 ++-- .../geotrek_parser_v2/infrastructure_ids.json | 4 +-- .../data/geotrek_parser_v2/signage_ids.json | 4 +-- .../touristiccontent_ids.json | 4 +-- .../geotrek_parser_v2/touristicevent_ids.json | 4 +-- geotrek/trekking/parsers.py | 6 ++-- .../tests/data/geotrek_parser_v2/trek.json | 2 +- .../data/geotrek_parser_v2/trek_children.json | 24 ++++++++-------- .../trek_children_do_not_exist.json | 28 +++++++++++-------- .../data/geotrek_parser_v2/trek_ids.json | 2 +- .../trek_wrong_children.json | 10 +++---- geotrek/trekking/tests/test_parsers.py | 4 +-- 12 files changed, 52 insertions(+), 46 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 2b5f7aea76..772ebdbaf5 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -1117,13 +1117,13 @@ def start(self): super().start() kwargs = self.get_to_delete_kwargs() kwargs['eid__startswith'] = self.eid_prefix + json_id_key = self.replace_fields.get('eid', 'id') params = { - 'fields': 'id', + 'fields': json_id_key, 'page_size': 10000 } - json_key = self.replace_fields.get('eid', 'id') response = self.request_or_retry(self.next_url, params=params) - ids = [f"{self.eid_prefix}{element[json_key]}" for element in response.json().get('results', [])] + ids = [f"{self.eid_prefix}{element[json_id_key]}" for element in response.json().get('results', [])] self.to_delete = set(self.model.objects.filter(**kwargs).exclude(eid__in=ids).values_list('pk', flat=True)) def filter_eid(self, src, val): diff --git a/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json index 220f595ded..5c5585bdb1 100644 --- a/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json +++ b/geotrek/infrastructure/tests/data/geotrek_parser_v2/infrastructure_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 2898 + "uuid": "93747e51-757e-4b39-9c3b-41bb228a7455" }, { - "id": 8385 + "uuid": "f7d10e86-5bf0-47a5-934e-63dcca8bf9c5" } ] } \ No newline at end of file diff --git a/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json b/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json index 2b648faf7a..ed3b697135 100644 --- a/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json +++ b/geotrek/signage/tests/data/geotrek_parser_v2/signage_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 8416 + "uuid": "6de1205c-8066-4e8b-8f84-f8736db61e95" }, { - "id": 8455 + "uuid": "133e9666-5bb5-4690-8034-6171bf4af115" } ] } \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json index 5e51ae2673..5252af41e0 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristiccontent_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 10 + "uuid": "921f9859-94b5-447a-ad20-946697323569" }, { - "id": 22 + "uuid": "a9bf823a-a472-4eb2-b273-3db1355602d0" } ] } \ No newline at end of file diff --git a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json index a935933a50..ab17c78dff 100644 --- a/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json +++ b/geotrek/tourism/tests/data/geotrek_parser_v2/touristicevent_ids.json @@ -4,10 +4,10 @@ "previous": null, "results": [ { - "id": 3 + "uuid": "7382c443-8206-4ef2-bf43-f471cdc5073d" }, { - "id": 1 + "uuid": "20fb113f-07f7-4deb-94b4-8445c85d5598" } ] } \ No newline at end of file diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 373e2201d6..9a87bc5aa7 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -122,17 +122,17 @@ def filter_points_reference(self, src, val): def end(self): """Add children after all treks imported are created in database.""" super().end() - self.next_url = f"{self.url}/api/v2/trek" + self.next_url = f"{self.url}/api/v2/tour" try: params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), - 'fields': 'children,id' + 'fields': 'steps,uuid' } response = self.request_or_retry(f"{self.next_url}", params=params) results = response.json()['results'] final_children = {} for result in results: - final_children[result['id']] = result['children'] + final_children[result['uuid']] = [step['uuid'] for step in result['steps']] for key, value in final_children.items(): if value: diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json index 2f5b4955ba..3abae5dd80 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -114,7 +114,7 @@ ], "attachments_accessibility": [], "children": [ - 10439 + "c9567576-2934-43ab-979e-e13d02c671a9" ], "cities": [ "09231", diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json index 3cb7aafe61..2f49e3edbf 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json @@ -4,26 +4,28 @@ "previous": null, "results": [ { - "id": 2, - "children": [ - 10439 + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1", + "steps": [ + { + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" + } ] }, { - "id": 10443, - "children": [] + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12", + "steps": [] }, { - "id": 2849, - "children": [] + "uuid": "6761143f-9244-41d0-b1af-21114408f769", + "steps": [] }, { - "id": 10439, - "children": [] + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9", + "steps": [] }, { - "id": 8700, - "children": [] + "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1", + "steps": [] } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json index 6fa8b5887c..948d0a0c73 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json @@ -4,27 +4,31 @@ "previous": null, "results": [ { - "id": 2, - "children": [ - 10439, - 10888 + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1", + "steps": [ + { + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" + }, + { + "uuid": "c9567576-2934-43ab-666e-e13d02c671a9" + } ] }, { - "id": 10443, - "children": [] + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12", + "steps": [] }, { - "id": 2849, - "children": [] + "uuid": "6761143f-9244-41d0-b1af-21114408f769", + "steps": [] }, { - "id": 10439, - "children": [] + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9", + "steps": [] }, { - "id": 8700, - "children": [] + "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1", + "steps": [] } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json index 62c9444a8e..4855289ff7 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json @@ -4,7 +4,7 @@ "previous": null, "results": [ { - "suuid": "9e70b294-1134-4c50-9c56-d722720cacf1" + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1" }, { "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12" diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json index 0c78b12976..4855289ff7 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json @@ -4,19 +4,19 @@ "previous": null, "results": [ { - "id": 2 + "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1" }, { - "id": 10443 + "uuid": "1ba24605-aff2-4b16-bf30-6de1ebfb2a12" }, { - "id": 2849 + "uuid": "6761143f-9244-41d0-b1af-21114408f769" }, { - "id": 10439 + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" }, { - "id": 8700 + "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1" } ] } \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 5ebd00c5da..449b196e55 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -219,7 +219,7 @@ def test_create_multiple_page(self, mocked_head, mocked_get): class MockResponse: mock_json_order = ['trek_difficulty.json', 'trek_route.json', 'trek_theme.json', 'trek_practice.json', 'trek_accessibility.json', 'trek_network.json', 'trek_label.json', 'trek_ids.json', - 'trek.json', 'trek_children.json', 'trek.json'] + 'trek.json', 'trek_children.json', 'trek_children.json'] mock_time = 0 total_mock_response = 1 @@ -393,7 +393,7 @@ def test_wrong_children_error(self, mocked_head, mocked_get): call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, stdout=output) - self.assertIn("An error occured in children generation : KeyError('children'", output.getvalue()) + self.assertIn("An error occured in children generation : KeyError('steps'", output.getvalue()) @mock.patch('requests.get') @mock.patch('requests.head') From 68192aa570f32d4421c3881ff85ff8e72e6ea929 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Mon, 12 Sep 2022 18:27:22 +0200 Subject: [PATCH 103/116] Change empty provider behaviour in parser --- geotrek/common/parsers.py | 9 +++++---- geotrek/common/tests/test_parsers.py | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 772ebdbaf5..51055ab33e 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -77,7 +77,7 @@ class Parser: warn_on_missing_objects = False separator = '+' eid = None - provider = "" + provider = None fields = None m2m_fields = {} constant_fields = {} @@ -266,7 +266,7 @@ def parse_obj(self, row, operation): self.add_warning(str(warnings)) return if operation == "created": - if hasattr(self.model, 'provider') and self.provider and not self.obj.provider: + if hasattr(self.model, 'provider') and self.provider is not None and not self.obj.provider: self.obj.provider = self.provider self.obj.save() else: @@ -310,7 +310,8 @@ def parse_row(self, row): self.add_warning(str(warnings)) return objects = self.model.objects.filter(**eid_kwargs) - if hasattr(self.model, 'provider'): + print(eid_kwargs) + if hasattr(self.model, 'provider') and self.provider is not None: objects = objects.filter(provider__exact=self.provider) if len(objects) == 0 and self.update_only: if self.warn_on_missing_objects: @@ -447,7 +448,7 @@ def get_to_delete_kwargs(self): kwargs[dst] = field.remote_field.model.objects.get(**filters) except field.remote_field.model.DoesNotExist: return None - if hasattr(self.model, 'provider'): + if hasattr(self.model, 'provider') and self.provider is not None: kwargs['provider__exact'] = self.provider return kwargs diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index c6ff7a11d3..49b040a07c 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -577,10 +577,9 @@ def test_delete_according_to_no_provider(self, mocked_get): self.assertEqual(t.provider, "") self.assertEqual(t.eid, "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") self.assertEqual(str(t.uuid), "58ed4fc1-645d-4bf6-b956-71f0a01a5eec") - t1 = TrekFactory(provider="Provider1", name="I should not be deleted", eid="1234") TrekFactory(provider="", name="I should be deleted", eid="12374") call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestNoProviderParser', verbosity=0) - self.assertEqual([t.pk, t1.pk], list(Trek.objects.values_list('pk', flat=True))) + self.assertEqual([t.pk], list(Trek.objects.values_list('pk', flat=True))) class GeotrekAggregatorParserTest(GeotrekParserTestMixin, TestCase): From 21796fe8c7fba5241a01f78738850dda826470b5 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Mon, 12 Sep 2022 18:29:34 +0200 Subject: [PATCH 104/116] Codestyle --- geotrek/common/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 51055ab33e..329d6d1531 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -266,7 +266,7 @@ def parse_obj(self, row, operation): self.add_warning(str(warnings)) return if operation == "created": - if hasattr(self.model, 'provider') and self.provider is not None and not self.obj.provider: + if hasattr(self.model, 'provider') and self.provider is not None and not self.obj.provider: self.obj.provider = self.provider self.obj.save() else: From 9c3e4f87e4ba6e65d6a4ee56559cccda92fee4a4 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Tue, 13 Sep 2022 10:49:12 +0200 Subject: [PATCH 105/116] Remove print --- geotrek/common/parsers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 329d6d1531..12c3501639 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -310,7 +310,6 @@ def parse_row(self, row): self.add_warning(str(warnings)) return objects = self.model.objects.filter(**eid_kwargs) - print(eid_kwargs) if hasattr(self.model, 'provider') and self.provider is not None: objects = objects.filter(provider__exact=self.provider) if len(objects) == 0 and self.update_only: From a3691911dd7e64ca30041c1cb0eb00c827cbe401 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Wed, 14 Sep 2022 16:41:39 +0200 Subject: [PATCH 106/116] Add end of files newlines --- .../geotrek_parser_v2/config_aggregator_no_data_to_import.json | 2 +- geotrek/common/tests/data/geotrek_parser_v2/treks.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/poi.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/service.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json | 2 +- .../tests/data/geotrek_parser_v2/trek_accessibility.json | 2 +- .../trekking/tests/data/geotrek_parser_v2/trek_children.json | 2 +- .../data/geotrek_parser_v2/trek_children_do_not_exist.json | 2 +- .../trekking/tests/data/geotrek_parser_v2/trek_difficulty.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json | 2 +- .../trekking/tests/data/geotrek_parser_v2/trek_practice.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json | 2 +- geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json | 2 +- .../tests/data/geotrek_parser_v2/trek_wrong_children.json | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json index 4b4195ff25..cf64abab46 100644 --- a/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json +++ b/geotrek/common/tests/data/geotrek_parser_v2/config_aggregator_no_data_to_import.json @@ -19,4 +19,4 @@ }, "create": true } -} \ No newline at end of file +} diff --git a/geotrek/common/tests/data/geotrek_parser_v2/treks.json b/geotrek/common/tests/data/geotrek_parser_v2/treks.json index 02b4e943c0..ba0d59a5b7 100644 --- a/geotrek/common/tests/data/geotrek_parser_v2/treks.json +++ b/geotrek/common/tests/data/geotrek_parser_v2/treks.json @@ -214,4 +214,4 @@ "web_links": [] } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json index a8d3b29f48..798a3999e2 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi.json @@ -136,4 +136,4 @@ "update_datetime": "2019-04-25T11:05:26.169812+02:00" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json index 470f590d2f..20ee702b48 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_ids.json @@ -10,4 +10,4 @@ "uuid": "542afd02-487a-4df7-afaf-2082b24e8b6e" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json index f8e2004c3c..b70131103a 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/poi_type.json @@ -164,4 +164,4 @@ "pictogram": "https://foo.fr/media/upload/poi-peak.png" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service.json index 985d25bfd2..94210bcc29 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/service.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service.json @@ -34,4 +34,4 @@ "uuid": "d2acb966-d01c-4eb1-81f9-a9f490c8484a" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json index 9183cf8824..a0f136a53b 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service_ids.json @@ -10,4 +10,4 @@ "uuid": "d2acb966-d01c-4eb1-81f9-a9f490c8484a" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json b/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json index 6ab2f72b26..4a5d79616b 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/service_type.json @@ -36,4 +36,4 @@ "pictogram": "https://foo.fr/media/upload/steep_descent.svg" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json index 3abae5dd80..daa3f5e8f9 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek.json @@ -1254,4 +1254,4 @@ "web_links": [] } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json index 82e8aa2989..639fdd9da5 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_2.json @@ -214,4 +214,4 @@ "web_links": [] } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json index 5d26de4998..1040952cd6 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_accessibility.json @@ -34,4 +34,4 @@ "pictogram": "https://foo.fr/media/upload/accessibility-joelette.png" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json index 2f49e3edbf..16226a3287 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json @@ -28,4 +28,4 @@ "steps": [] } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json index 948d0a0c73..06acc25869 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json @@ -31,4 +31,4 @@ "steps": [] } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json index d361cd9c5a..da62db310b 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_difficulty.json @@ -59,4 +59,4 @@ "pictogram": "https://foo.fr/media/upload/difficulty-5.svg" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json index 4855289ff7..a6ab1d4325 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids.json @@ -19,4 +19,4 @@ "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json index 8541f394b4..35c48d9a9b 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_ids_2.json @@ -10,4 +10,4 @@ "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json index cb5aecc593..67f0fa464a 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_label.json @@ -55,4 +55,4 @@ "pictogram": null } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json index 7548a9b966..cdff494575 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json @@ -24,4 +24,4 @@ "pictogram": "https://foo.fr/media/upload/network-VTT.svg" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json index 9f0d40c272..ac28443b96 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_practice.json @@ -37,4 +37,4 @@ "pictogram": "https://foo.fr/media/upload/practice-foot.svg" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json index 0deba02309..17e5a389db 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json @@ -34,4 +34,4 @@ } } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json index cc77967cf7..bb6570f322 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_theme.json @@ -104,4 +104,4 @@ "pictogram": "https://foo.fr/media/upload/theme-history.png" } ] -} \ No newline at end of file +} diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json index 4855289ff7..a6ab1d4325 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_wrong_children.json @@ -19,4 +19,4 @@ "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1" } ] -} \ No newline at end of file +} From b16af28d7addb549e0499b3b17e686706e0a23ca Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Thu, 15 Sep 2022 11:19:01 +0200 Subject: [PATCH 107/116] Replace 'eid_prefix' with 'provider' in GeotrekParser --- geotrek/common/parsers.py | 25 ++++++++++--------------- geotrek/trekking/parsers.py | 4 ++-- geotrek/trekking/tests/test_parsers.py | 4 ++-- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 12c3501639..23e604a809 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -970,7 +970,7 @@ def parse(self, filename=None, limit=None): key_warning = _("Geotrek-admin") self.add_warning(key_warning, warning) else: - Parser = parser(progress_cb=self.progress_cb, eid_prefix=key, url=datas['url'], + Parser = parser(progress_cb=self.progress_cb, provider=key, url=datas['url'], portals_filter=datas.get('portals'), mapping=datas.get('mapping'), create_categories=datas.get('create'), all_datas=datas.get('all_datas')) self.progress_cb(0, 0, f'{model} ({key})') @@ -997,7 +997,7 @@ class GeotrekParser(AttachmentParserMixin, Parser): replace_fields: Replace fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) m2m_replace_fields: Replace m2m fields which have not the same name in the api v2 compare to models (geom => geometry in api v2) categories_keys_api_v2: Key in the route of the category (example: /api/v2/touristiccontent_category/) corresponding to the model field - eid_prefix: Prefix of your eid which allow to differentiate multiple GeotrekParser + provider: Allow to differentiate multiple GeotrekParser for the same model portals_filter: Portals which will be use for filter in api v2 (default: No portal filter) mapping: Mapping between values in categories (example: /api/v2/touristiccontent_category/) and final values Can be use when you want to change a value from the api/v2 @@ -1022,13 +1022,13 @@ class GeotrekParser(AttachmentParserMixin, Parser): 'geom': {'required': True}, } bbox = None - eid_prefix = '' portals_filter = None mapping = {} create_categories = False all_datas = False + provider = None - def __init__(self, all_datas=None, create_categories=None, eid_prefix=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): + def __init__(self, all_datas=None, create_categories=None, provider=None, mapping=None, portals_filter=None, url=None, *args, **kwargs): super().__init__(*args, **kwargs) self.bbox = Polygon.from_bbox(settings.SPATIAL_EXTENT) self.bbox.srid = settings.SRID @@ -1036,10 +1036,10 @@ def __init__(self, all_datas=None, create_categories=None, eid_prefix=None, mapp self.portals_filter = portals_filter self.url = url if url else self.url self.mapping = mapping if mapping else self.mapping - self.eid_prefix = eid_prefix if eid_prefix else self.eid_prefix + self.provider = provider if provider else self.provider self.all_datas = all_datas if all_datas else self.all_datas self.create_categories = create_categories if create_categories else self.create_categories - self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and not f.name == 'id') + self.fields = dict((f.name, f.name) for f in self.model._meta.fields if not isinstance(f, TranslationField) and f.name != 'id') self.m2m_fields = { f.name: f.name for f in self.model._meta.many_to_many @@ -1116,19 +1116,15 @@ def generate_attachment(self, **kwargs): def start(self): super().start() kwargs = self.get_to_delete_kwargs() - kwargs['eid__startswith'] = self.eid_prefix json_id_key = self.replace_fields.get('eid', 'id') params = { 'fields': json_id_key, 'page_size': 10000 } response = self.request_or_retry(self.next_url, params=params) - ids = [f"{self.eid_prefix}{element[json_id_key]}" for element in response.json().get('results', [])] + ids = [f"{element[json_id_key]}" for element in response.json().get('results', [])] self.to_delete = set(self.model.objects.filter(**kwargs).exclude(eid__in=ids).values_list('pk', flat=True)) - def filter_eid(self, src, val): - return f'{self.eid_prefix}{val}' - def filter_attachments(self, src, val): return [(subval.get('url'), subval.get('legend'), subval.get('author'), subval.get('license')) for subval in val] @@ -1164,10 +1160,9 @@ def next_row(self): portals = self.portals_filter updated_after = None - if not self.all_datas and self.model.objects.filter(eid__startswith=self.eid_prefix).exists() and 'date_update' in [field.name for - field in - self.model._meta.get_fields()]: - updated_after = self.model.objects.filter(eid__startswith=self.eid_prefix).latest('date_update').date_update.strftime('%Y-%m-%d') + available_fields = [field.name for field in self.model._meta.get_fields()] + if not self.all_datas and self.model.objects.filter(provider__exact=self.provider).exists() and 'date_update' in available_fields: + updated_after = self.model.objects.filter(provider__exact=self.provider).latest('date_update').date_update.strftime('%Y-%m-%d') params = { 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]), 'portals': ','.join(portals) if portals else '', diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 9a87bc5aa7..e63d570e91 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -136,13 +136,13 @@ def end(self): for key, value in final_children.items(): if value: - trek_parent_instance = Trek.objects.filter(eid=f"{self.eid_prefix}{key}") + trek_parent_instance = Trek.objects.filter(eid=key) if not trek_parent_instance: return order = 0 for child in value: try: - trek_child_instance = Trek.objects.get(eid=f"{self.eid_prefix}{child}") + trek_child_instance = Trek.objects.get(eid=child) except Trek.DoesNotExist: self.add_warning(_(f"One trek has not be generated for {trek_parent_instance[0].name}")) continue diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 449b196e55..413deb3ea0 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -135,7 +135,7 @@ def test_create(self): class TestGeotrekTrekParser(GeotrekTrekParser): url = "https://test.fr" - eid_prefix = 'geotrek1' + provider = 'geotrek1' field_options = { 'difficulty': {'create': True, }, 'route': {'create': True, }, @@ -154,7 +154,7 @@ class TestGeotrek2TrekParser(GeotrekTrekParser): field_options = { 'geom': {'required': True}, } - eid_prefix = 'geotrek2' + provider = 'geotrek2' class TestGeotrekPOIParser(GeotrekPOIParser): From 2ef70b5b8ce835b0337fb5e20bf08f7fedadeecd Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Thu, 15 Sep 2022 16:40:54 +0200 Subject: [PATCH 108/116] Improve error message in Tour parsing --- geotrek/trekking/parsers.py | 2 +- geotrek/trekking/tests/test_parsers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index e63d570e91..b9835a661c 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -144,7 +144,7 @@ def end(self): try: trek_child_instance = Trek.objects.get(eid=child) except Trek.DoesNotExist: - self.add_warning(_(f"One trek has not be generated for {trek_parent_instance[0].name}")) + self.add_warning(_(f"One trek has not be generated for {trek_parent_instance[0].name} : could not find trek with UUID {child}")) continue OrderedTrekChild.objects.get_or_create(parent=trek_parent_instance[0], child=trek_child_instance, diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 413deb3ea0..e15f54c2e2 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -374,7 +374,7 @@ def test_children_do_not_exist(self, mocked_head, mocked_get): output = StringIO() call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, stdout=output) - self.assertIn("One trek has not be generated for Boucle du Pic des Trois Seigneurs", output.getvalue()) + self.assertIn("One trek has not be generated for Boucle du Pic des Trois Seigneurs : could not find trek with UUID c9567576-2934-43ab-666e-e13d02c671a9,\n", output.getvalue()) @mock.patch('requests.get') @mock.patch('requests.head') From 9a6058d599b62e1d7f165b6c3899bea97419aebc Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 16 Sep 2022 16:38:47 +0200 Subject: [PATCH 109/116] Add missing provider filters --- geotrek/core/filters.py | 18 +++++++++++++++--- geotrek/core/models.py | 14 ++++++++++++++ geotrek/outdoor/filters.py | 18 +++++++++++++++--- geotrek/outdoor/models.py | 19 +++++++++++++++++++ geotrek/sensitivity/filters.py | 12 +++++++++--- geotrek/sensitivity/models.py | 10 ++++++++++ geotrek/trekking/serializers.py | 2 +- 7 files changed, 83 insertions(+), 10 deletions(-) diff --git a/geotrek/core/filters.py b/geotrek/core/filters.py index 334b4892d6..d9490ecc70 100644 --- a/geotrek/core/filters.py +++ b/geotrek/core/filters.py @@ -1,7 +1,7 @@ from django.conf import settings from django.db.models import Count, F, Q from django.utils.translation import gettext_lazy as _ -from django_filters import BooleanFilter, CharFilter, FilterSet, ModelMultipleChoiceFilter +from django_filters import BooleanFilter, CharFilter, FilterSet, ModelMultipleChoiceFilter, ChoiceFilter from .models import Topology, Path, Trail, CertificationLabel @@ -99,11 +99,17 @@ def _topology_filter(self, qs, edges): class PathFilterSet(AltimetryAllGeometriesFilterSet, ZoningFilterSet, StructureRelatedFilterSet): name = CharFilter(label=_('Name'), lookup_expr='icontains') comments = CharFilter(label=_('Comments'), lookup_expr='icontains') + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=Path.objects.provider_choices() + ) class Meta(StructureRelatedFilterSet.Meta): model = Path fields = StructureRelatedFilterSet.Meta.fields + \ - ['valid', 'networks', 'usages', 'comfort', 'stake', 'draft', ] + ['valid', 'networks', 'usages', 'comfort', 'stake', 'draft', 'provider'] class TrailFilterSet(AltimetryAllGeometriesFilterSet, ValidTopologyFilterSet, ZoningFilterSet, StructureRelatedFilterSet): @@ -117,11 +123,17 @@ class TrailFilterSet(AltimetryAllGeometriesFilterSet, ValidTopologyFilterSet, Zo label=_("Certification labels"), queryset=CertificationLabel.objects.all(), ) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=Trail.objects.provider_choices() + ) class Meta(StructureRelatedFilterSet.Meta): model = Trail fields = StructureRelatedFilterSet.Meta.fields + \ - ['name', 'category', 'departure', 'arrival', 'certification_labels', 'comments'] + ['name', 'category', 'departure', 'arrival', 'certification_labels', 'comments', 'provider'] class TopologyFilterTrail(TopologyFilter): diff --git a/geotrek/core/models.py b/geotrek/core/models.py index 7794c42564..6bbf02c811 100644 --- a/geotrek/core/models.py +++ b/geotrek/core/models.py @@ -47,6 +47,11 @@ def get_queryset(self): """ return super().get_queryset().filter(visible=True).annotate(length_2d=Length('geom')) + def provider_choices(self): + providers = self.get_queryset().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + class PathInvisibleManager(models.Manager): use_for_related_fields = True @@ -957,6 +962,13 @@ def __str__(self): return self.network +class TrailManager(TopologyManager): + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='').order_by('provider') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class Trail(MapEntityMixin, Topology, StructureRelated): topo_object = models.OneToOneField(Topology, parent_link=True, on_delete=models.CASCADE) name = models.CharField(verbose_name=_("Name"), max_length=64) @@ -976,6 +988,8 @@ class Trail(MapEntityMixin, Topology, StructureRelated): certifications_verbose_name = _("Certifications") geometry_types_allowed = ["LINESTRING"] + objects = TrailManager() + class Meta: verbose_name = _("Trail") verbose_name_plural = _("Trails") diff --git a/geotrek/outdoor/filters.py b/geotrek/outdoor/filters.py index 454ace0b6a..baac218c62 100644 --- a/geotrek/outdoor/filters.py +++ b/geotrek/outdoor/filters.py @@ -1,6 +1,6 @@ from django.db.models import Q from django.utils.translation import gettext as _ -from django_filters.filters import MultipleChoiceFilter, ModelMultipleChoiceFilter +from django_filters.filters import MultipleChoiceFilter, ModelMultipleChoiceFilter, ChoiceFilter from geotrek.authent.filters import StructureRelatedFilterSet from geotrek.common.models import Organism from geotrek.outdoor.models import Site, Practice, Sector, Course @@ -13,12 +13,18 @@ class SiteFilterSet(ZoningFilterSet, StructureRelatedFilterSet): practice = ModelMultipleChoiceFilter(queryset=Practice.objects.all(), method='filter_super') sector = ModelMultipleChoiceFilter(queryset=Sector.objects.all(), method='filter_sector', label=_("Sector")) managers = ModelMultipleChoiceFilter(queryset=Organism.objects.all(), method='filter_manager', label=_("Manager")) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=Site.objects.provider_choices() + ) class Meta(StructureRelatedFilterSet.Meta): model = Site fields = StructureRelatedFilterSet.Meta.fields + [ 'sector', 'practice', 'labels', 'themes', 'portal', 'source', 'information_desks', - 'web_links', 'type', 'orientation', 'wind', + 'web_links', 'type', 'orientation', 'wind', 'provider' ] def filter_orientation(self, qs, name, values): @@ -48,13 +54,19 @@ class CourseFilterSet(ZoningFilterSet, StructureRelatedFilterSet): label=_("Orientation")) wind = MultipleChoiceFilter(choices=Site.WIND_CHOICES, method='filter_orientation', label=_("Wind")) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=Course.objects.provider_choices() + ) class Meta(StructureRelatedFilterSet.Meta): model = Course fields = StructureRelatedFilterSet.Meta.fields + [ 'parent_sites', 'parent_sites__practice__sector', 'parent_sites__practice', 'parent_sites__labels', 'parent_sites__themes', 'parent_sites__portal', 'parent_sites__source', 'parent_sites__type', 'orientation', 'wind', - 'height', + 'height', 'provider' ] def filter_orientation(self, qs, name, values): diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index 339ab8caf1..75360d8edf 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -26,6 +26,7 @@ from geotrek.trekking.models import POI, Service, Trek from geotrek.zoning.mixins import ZoningPropertiesMixin from mapentity.models import MapEntityMixin +from modeltranslation.manager import MultilingualManager class AltimetryMixin(BaseAltimetryMixin): @@ -110,6 +111,13 @@ def __str__(self): return self.name +class SiteManager(MultilingualManager): + def provider_choices(self): + providers = self.get_queryset().exclude(provider__exact='').order_by('provider') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class Site(ZoningPropertiesMixin, AddPropertyMixin, PicturesMixin, PublishableMixin, MapEntityMixin, StructureRelated, AltimetryMixin, TimeStampedModelMixin, MPTTModel, ExcludedPOIsMixin): ORIENTATION_CHOICES = ( @@ -177,6 +185,8 @@ class Site(ZoningPropertiesMixin, AddPropertyMixin, PicturesMixin, PublishableMi check_structure_in_forms = False + objects = SiteManager() + class Meta: verbose_name = _("Outdoor site") verbose_name_plural = _("Outdoor sites") @@ -338,6 +348,13 @@ class Meta: ) +class CourseManager(MultilingualManager): + def provider_choices(self): + providers = self.get_queryset().exclude(provider__exact='').order_by('provider') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class Course(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, MapEntityMixin, StructureRelated, PicturesMixin, AltimetryMixin, TimeStampedModelMixin, ExcludedPOIsMixin): geom = models.GeometryCollectionField(verbose_name=_("Location"), srid=settings.SRID) @@ -367,6 +384,8 @@ class Course(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, MapEntit check_structure_in_forms = False + objects = CourseManager() + class Meta: verbose_name = _("Outdoor course") verbose_name_plural = _("Outdoor courses") diff --git a/geotrek/sensitivity/filters.py b/geotrek/sensitivity/filters.py index 1e8fa26a4c..0b631e1b8c 100644 --- a/geotrek/sensitivity/filters.py +++ b/geotrek/sensitivity/filters.py @@ -1,5 +1,5 @@ -from django.utils.translation import pgettext_lazy -from django_filters.filters import ModelMultipleChoiceFilter +from django.utils.translation import pgettext_lazy, gettext as _ +from django_filters.filters import ModelMultipleChoiceFilter, ChoiceFilter from geotrek.authent.filters import StructureRelatedFilterSet from .models import SensitiveArea, Species @@ -9,9 +9,15 @@ class SensitiveAreaFilterSet(StructureRelatedFilterSet): label=pgettext_lazy("Singular", "Species"), queryset=Species.objects.filter(category=Species.SPECIES) ) + provider = ChoiceFilter( + field_name='provider', + empty_label=_("Provider"), + label=_("Provider"), + choices=SensitiveArea.objects.provider_choices() + ) class Meta(StructureRelatedFilterSet.Meta): model = SensitiveArea fields = StructureRelatedFilterSet.Meta.fields + [ - 'species', 'species__category', + 'species', 'species__category', 'provider' ] diff --git a/geotrek/sensitivity/models.py b/geotrek/sensitivity/models.py index 03eb19f7c1..16066d4ce9 100644 --- a/geotrek/sensitivity/models.py +++ b/geotrek/sensitivity/models.py @@ -3,6 +3,7 @@ """ import datetime +from geotrek.common.mixins.managers import NoDeleteManager import simplekml from django.conf import settings @@ -72,6 +73,13 @@ def pretty_practices(self): return ", ".join([str(practice) for practice in self.practices.all()]) +class SensitiveAreaManager(NoDeleteManager): + def provider_choices(self): + providers = self.get_queryset().existing().exclude(provider__exact='') \ + .distinct('provider').values_list('provider', 'provider') + return providers + + class SensitiveArea(MapEntityMixin, StructureRelated, TimeStampedModelMixin, NoDeleteMixin, AddPropertyMixin): geom = models.GeometryField(srid=settings.SRID) @@ -83,6 +91,8 @@ class SensitiveArea(MapEntityMixin, StructureRelated, TimeStampedModelMixin, NoD eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + objects = SensitiveAreaManager() + class Meta: verbose_name = _("Sensitive area") verbose_name_plural = _("Sensitive areas") diff --git a/geotrek/trekking/serializers.py b/geotrek/trekking/serializers.py index aa7529d25f..b6b0c9e4e4 100644 --- a/geotrek/trekking/serializers.py +++ b/geotrek/trekking/serializers.py @@ -363,7 +363,7 @@ class POIAPISerializer(PublishableSerializerMixin, PicturesSerializerMixin, Zoni structure = StructureSerializer() class Meta: - model = trekking_models.Trek + model = trekking_models.POI id_field = 'id' # By default on this model it's topo_object = OneToOneField(parent_link=True) fields = ( 'id', 'description', 'type', 'min_elevation', 'max_elevation', 'structure' From 84d8af632895e5dab3b6a8bda3cbbe2bb0cf790b Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 16 Sep 2022 17:11:13 +0200 Subject: [PATCH 110/116] Add db index on providers --- geotrek/core/migrations/0034_auto_20220909_1316.py | 4 ++-- geotrek/core/models.py | 4 ++-- .../migrations/0031_infrastructure_provider.py | 2 +- geotrek/infrastructure/models.py | 2 +- geotrek/outdoor/migrations/0040_auto_20220909_1327.py | 4 ++-- geotrek/outdoor/models.py | 4 ++-- .../sensitivity/migrations/0022_sensitivearea_provider.py | 2 +- geotrek/sensitivity/models.py | 2 +- geotrek/signage/migrations/0024_signage_provider.py | 2 +- geotrek/tourism/migrations/0026_auto_20220907_1400.py | 6 +++--- geotrek/tourism/models.py | 6 +++--- geotrek/trekking/migrations/0042_auto_20220907_1253.py | 6 +++--- geotrek/trekking/models.py | 6 +++--- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/geotrek/core/migrations/0034_auto_20220909_1316.py b/geotrek/core/migrations/0034_auto_20220909_1316.py index b7db1254fa..6f565a47f5 100644 --- a/geotrek/core/migrations/0034_auto_20220909_1316.py +++ b/geotrek/core/migrations/0034_auto_20220909_1316.py @@ -13,11 +13,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='path', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), migrations.AddField( model_name='trail', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/core/models.py b/geotrek/core/models.py index 6bbf02c811..d8325fd220 100644 --- a/geotrek/core/models.py +++ b/geotrek/core/models.py @@ -96,7 +96,7 @@ class Path(ZoningPropertiesMixin, AddPropertyMixin, MapEntityMixin, AltimetryMix blank=True, related_name="paths", verbose_name=_("Networks")) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) draft = models.BooleanField(default=False, verbose_name=_("Draft"), db_index=True) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) @@ -983,7 +983,7 @@ class Trail(MapEntityMixin, Topology, StructureRelated): arrival = models.CharField(verbose_name=_("Arrival"), blank=True, max_length=64) comments = models.TextField(default="", blank=True, verbose_name=_("Comments")) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) certifications_verbose_name = _("Certifications") geometry_types_allowed = ["LINESTRING"] diff --git a/geotrek/infrastructure/migrations/0031_infrastructure_provider.py b/geotrek/infrastructure/migrations/0031_infrastructure_provider.py index 58e08fd3b3..a6b7576526 100644 --- a/geotrek/infrastructure/migrations/0031_infrastructure_provider.py +++ b/geotrek/infrastructure/migrations/0031_infrastructure_provider.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='infrastructure', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/infrastructure/models.py b/geotrek/infrastructure/models.py index cfbf8436ef..6c89156efb 100755 --- a/geotrek/infrastructure/models.py +++ b/geotrek/infrastructure/models.py @@ -99,7 +99,7 @@ class BaseInfrastructure(BasePublishableMixin, Topology, StructureRelated): on_delete=models.SET_NULL) implantation_year = models.PositiveSmallIntegerField(verbose_name=_("Implantation year"), null=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) class Meta: abstract = True diff --git a/geotrek/outdoor/migrations/0040_auto_20220909_1327.py b/geotrek/outdoor/migrations/0040_auto_20220909_1327.py index e37f9b94d1..4da4f9b5d6 100644 --- a/geotrek/outdoor/migrations/0040_auto_20220909_1327.py +++ b/geotrek/outdoor/migrations/0040_auto_20220909_1327.py @@ -13,11 +13,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='course', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), migrations.AddField( model_name='site', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index 75360d8edf..f649332968 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -179,7 +179,7 @@ class Site(ZoningPropertiesMixin, AddPropertyMixin, PicturesMixin, PublishableMi type = models.ForeignKey(SiteType, related_name="sites", on_delete=models.PROTECT, verbose_name=_("Type"), null=True, blank=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) managers = models.ManyToManyField(Organism, verbose_name=_("Managers"), blank=True) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) @@ -373,7 +373,7 @@ class Course(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, MapEntit ratings = models.ManyToManyField(Rating, related_name='courses', blank=True, verbose_name=_("Ratings")) height = models.IntegerField(verbose_name=_("Height"), blank=True, null=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) type = models.ForeignKey(CourseType, related_name="courses", on_delete=models.PROTECT, verbose_name=_("Type"), null=True, blank=True) pois_excluded = models.ManyToManyField('trekking.Poi', related_name='excluded_courses', verbose_name=_("Excluded POIs"), diff --git a/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py b/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py index 53c8da720b..318960a4f8 100644 --- a/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py +++ b/geotrek/sensitivity/migrations/0022_sensitivearea_provider.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='sensitivearea', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/sensitivity/models.py b/geotrek/sensitivity/models.py index 16066d4ce9..f02b01d2ff 100644 --- a/geotrek/sensitivity/models.py +++ b/geotrek/sensitivity/models.py @@ -89,7 +89,7 @@ class SensitiveArea(MapEntityMixin, StructureRelated, TimeStampedModelMixin, NoD description = models.TextField(verbose_name=_("Description"), blank=True) contact = models.TextField(verbose_name=_("Contact"), blank=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) objects = SensitiveAreaManager() diff --git a/geotrek/signage/migrations/0024_signage_provider.py b/geotrek/signage/migrations/0024_signage_provider.py index 854d307e9f..90eb791f7b 100644 --- a/geotrek/signage/migrations/0024_signage_provider.py +++ b/geotrek/signage/migrations/0024_signage_provider.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='signage', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/tourism/migrations/0026_auto_20220907_1400.py b/geotrek/tourism/migrations/0026_auto_20220907_1400.py index 0b88fc9db5..4d6e9a5e93 100644 --- a/geotrek/tourism/migrations/0026_auto_20220907_1400.py +++ b/geotrek/tourism/migrations/0026_auto_20220907_1400.py @@ -13,16 +13,16 @@ class Migration(migrations.Migration): migrations.AddField( model_name='informationdesk', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), migrations.AddField( model_name='touristiccontent', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), migrations.AddField( model_name='touristicevent', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/tourism/models.py b/geotrek/tourism/models.py index 0dfd2b2732..4cb85eb72a 100644 --- a/geotrek/tourism/models.py +++ b/geotrek/tourism/models.py @@ -60,7 +60,7 @@ def __str__(self): class InformationDesk(models.Model): eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) name = models.CharField(verbose_name=_("Title"), max_length=256) type = models.ForeignKey(InformationDeskType, verbose_name=_("Type"), on_delete=models.CASCADE, related_name='desks') @@ -322,7 +322,7 @@ class TouristicContent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin on_delete=models.CASCADE, related_name='contents', blank=True, null=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) reservation_system = models.ForeignKey(ReservationSystem, verbose_name=_("Reservation system"), on_delete=models.CASCADE, blank=True, null=True) reservation_id = models.CharField(verbose_name=_("Reservation ID"), max_length=1024, @@ -444,7 +444,7 @@ class TouristicEvent(ZoningPropertiesMixin, AddPropertyMixin, PublishableMixin, blank=True, related_name='touristicevents', verbose_name=_("Portal")) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) approved = models.BooleanField(verbose_name=_("Approved"), default=False) uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) objects = TouristicEventManager() diff --git a/geotrek/trekking/migrations/0042_auto_20220907_1253.py b/geotrek/trekking/migrations/0042_auto_20220907_1253.py index 3edd3b315c..88064e7a5a 100644 --- a/geotrek/trekking/migrations/0042_auto_20220907_1253.py +++ b/geotrek/trekking/migrations/0042_auto_20220907_1253.py @@ -13,16 +13,16 @@ class Migration(migrations.Migration): migrations.AddField( model_name='poi', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), migrations.AddField( model_name='service', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), migrations.AddField( model_name='trek', name='provider', - field=models.CharField(blank=True, max_length=1024, verbose_name='Provider'), + field=models.CharField(blank=True, db_index=True, max_length=1024, verbose_name='Provider'), ), ] diff --git a/geotrek/trekking/models.py b/geotrek/trekking/models.py index 9b17303a24..8d6f06a1d8 100755 --- a/geotrek/trekking/models.py +++ b/geotrek/trekking/models.py @@ -209,7 +209,7 @@ class Trek(Topology, StructureRelated, PicturesMixin, PublishableMixin, MapEntit verbose_name=_("Labels"), blank=True) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) eid2 = models.CharField(verbose_name=_("Second external id"), max_length=1024, blank=True, null=True) pois_excluded = models.ManyToManyField('Poi', related_name='excluded_treks', verbose_name=_("Excluded POIs"), blank=True) @@ -738,7 +738,7 @@ class POI(StructureRelated, PicturesMixin, PublishableMixin, MapEntityMixin, Top description = models.TextField(verbose_name=_("Description"), blank=True, help_text=_("History, details, ...")) type = models.ForeignKey('POIType', related_name='pois', verbose_name=_("Type"), on_delete=models.CASCADE) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) geometry_types_allowed = ["POINT"] @@ -875,7 +875,7 @@ class Service(StructureRelated, MapEntityMixin, Topology): on_delete=models.CASCADE) type = models.ForeignKey('ServiceType', related_name='services', verbose_name=_("Type"), on_delete=models.CASCADE) eid = models.CharField(verbose_name=_("External id"), max_length=1024, blank=True, null=True) - provider = models.CharField(verbose_name=_("Provider"), max_length=1024, blank=True) + provider = models.CharField(verbose_name=_("Provider"), db_index=True, max_length=1024, blank=True) class Meta: verbose_name = _("Service") From 9be78993fa012e1046060fb189718614be7225df Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 16 Sep 2022 17:24:20 +0200 Subject: [PATCH 111/116] Fix Outdoor Managers --- geotrek/outdoor/models.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index f649332968..bcfb816f94 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -5,10 +5,10 @@ from django.contrib.gis.measure import D from django.contrib.postgres.indexes import GistIndex from django.core.validators import MinValueValidator -from django.db.models import Q +from django.db.models import Q, Manager from django.utils.html import escape from django.utils.translation import gettext_lazy as _ -from mptt.models import MPTTModel, TreeForeignKey +from mptt.models import MPTTModel, TreeForeignKey, TreeManager from geotrek.altimetry.models import AltimetryMixin as BaseAltimetryMixin from geotrek.authent.models import StructureRelated @@ -26,7 +26,6 @@ from geotrek.trekking.models import POI, Service, Trek from geotrek.zoning.mixins import ZoningPropertiesMixin from mapentity.models import MapEntityMixin -from modeltranslation.manager import MultilingualManager class AltimetryMixin(BaseAltimetryMixin): @@ -111,7 +110,7 @@ def __str__(self): return self.name -class SiteManager(MultilingualManager): +class SiteManager(TreeManager): def provider_choices(self): providers = self.get_queryset().exclude(provider__exact='').order_by('provider') \ .distinct('provider').values_list('provider', 'provider') @@ -348,7 +347,7 @@ class Meta: ) -class CourseManager(MultilingualManager): +class CourseManager(Manager): def provider_choices(self): providers = self.get_queryset().exclude(provider__exact='').order_by('provider') \ .distinct('provider').values_list('provider', 'provider') From 45e4f9a7b70965e82f53eb69652b6ae12f8d5c32 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 16 Sep 2022 18:14:28 +0200 Subject: [PATCH 112/116] Fix tests with new queries --- geotrek/core/tests/test_views.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/geotrek/core/tests/test_views.py b/geotrek/core/tests/test_views.py index 3a437630df..f0e031db88 100644 --- a/geotrek/core/tests/test_views.py +++ b/geotrek/core/tests/test_views.py @@ -575,7 +575,7 @@ def test_draft_path_layer_cache(self): self.modelfactory(draft=True) # There are 7 queries to get layer without drafts - with self.assertNumQueries(5): + with self.assertNumQueries(6): response = self.client.get(obj.get_layer_url(), {"_no_draft": "true"}) self.assertEqual(len(response.json()['features']), 1) @@ -604,7 +604,7 @@ def test_draft_path_layer_cache(self): self.modelfactory(draft=False) # Cache was updated, the path was not a draft : we get 7 queries - with self.assertNumQueries(5): + with self.assertNumQueries(6): self.client.get(obj.get_layer_url(), {"_no_draft": "true"}) def test_path_layer_cache(self): @@ -618,7 +618,7 @@ def test_path_layer_cache(self): self.modelfactory(draft=True) # There are 7 queries to get layer without drafts - with self.assertNumQueries(5): + with self.assertNumQueries(6): response = self.client.get(obj.get_layer_url()) self.assertEqual(len(response.json()['features']), 2) @@ -641,13 +641,13 @@ def test_path_layer_cache(self): self.modelfactory(draft=True) # Cache is updated when we add a draft path - with self.assertNumQueries(5): + with self.assertNumQueries(6): self.client.get(obj.get_layer_url()) self.modelfactory(draft=False) # Cache is updated when we add a path - with self.assertNumQueries(5): + with self.assertNumQueries(6): self.client.get(obj.get_layer_url()) @@ -700,7 +700,7 @@ def test_denormalized_path_trails(self): PathFactory.create_batch(size=50) TrailFactory.create_batch(size=50) self.login() - with self.assertNumQueries(6): + with self.assertNumQueries(7): self.client.get(reverse('core:path-drf-list', kwargs={'format': 'datatables'})) @@ -803,7 +803,7 @@ def test_add_trail_from_existing_topology(self): def test_perfs_export_csv(self): self.modelfactory.create() - with self.assertNumQueries(10): + with self.assertNumQueries(13): self.client.get(self.model.get_format_list_url() + '?format=csv') From 3684194a69bfae27381fefb6f5036562ae2ec50a Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Fri, 16 Sep 2022 18:53:57 +0200 Subject: [PATCH 113/116] Start documenting Parser attributes --- geotrek/common/parsers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 23e604a809..2726202603 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -65,6 +65,9 @@ class DownloadImportError(ImportError): class Parser: + """ + provider: Allow to differentiate multiple GeotrekParser for the same model + """ label = None model = None filename = None From 0ac86f15eab747ef31ab2a6394321be8746c4817 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Mon, 19 Sep 2022 15:31:40 +0200 Subject: [PATCH 114/116] Alter proper table in SQL templates for Signage --- geotrek/signage/templates/signage/sql/post_90_defaults.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geotrek/signage/templates/signage/sql/post_90_defaults.sql b/geotrek/signage/templates/signage/sql/post_90_defaults.sql index 93c854ab9e..749ea89dbc 100644 --- a/geotrek/signage/templates/signage/sql/post_90_defaults.sql +++ b/geotrek/signage/templates/signage/sql/post_90_defaults.sql @@ -19,14 +19,14 @@ ALTER TABLE signage_signage ALTER COLUMN code SET DEFAULT ''; --type -- topo_object -- name -ALTER TABLE infrastructure_infrastructure ALTER COLUMN description SET DEFAULT ''; +ALTER TABLE signage_signage ALTER COLUMN description SET DEFAULT ''; -- condition -- implantation_year -- eid -ALTER TABLE infrastructure_infrastructure ALTER COLUMN published SET DEFAULT FALSE; +ALTER TABLE signage_signage ALTER COLUMN published SET DEFAULT FALSE; -- publication_date -- structure -ALTER TABLE infrastructure_infrastructure ALTER COLUMN provider SET DEFAULT ''; +ALTER TABLE signage_signage ALTER COLUMN provider SET DEFAULT ''; From c92eec9c21fa2fb065549cdfe2d3e36d64389131 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Mon, 19 Sep 2022 16:01:34 +0200 Subject: [PATCH 115/116] Fix test should not depend on list order --- geotrek/common/tests/test_parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/tests/test_parsers.py b/geotrek/common/tests/test_parsers.py index 49b040a07c..33bd69f0fc 100644 --- a/geotrek/common/tests/test_parsers.py +++ b/geotrek/common/tests/test_parsers.py @@ -564,7 +564,7 @@ def test_delete_according_to_provider(self, mocked_get): t2 = TrekFactory(provider="Provider2", name="I should not be deleted", eid="1236") t3 = TrekFactory(provider="", name="I should not be deleted", eid="12374") call_command('import', 'geotrek.common.tests.test_parsers.GeotrekTrekTestProviderParser', verbosity=0) - self.assertEqual([t.pk, t2.pk, t3.pk], list(Trek.objects.values_list('pk', flat=True))) + self.assertListEqual([t.pk, t2.pk, t3.pk], list(Trek.objects.values_list('pk', flat=True))) @mock.patch('requests.get') def test_delete_according_to_no_provider(self, mocked_get): From b72044e6da72cd0bada9bbd70c566e243a5bb667 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Mon, 19 Sep 2022 17:28:17 +0200 Subject: [PATCH 116/116] Add warning and test for missing Tour parent trek --- geotrek/trekking/parsers.py | 1 + .../geotrek_parser_v2/trek_children_do_not_exist.json | 8 ++++++++ geotrek/trekking/tests/test_parsers.py | 1 + 3 files changed, 10 insertions(+) diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index b9835a661c..8448182a32 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -138,6 +138,7 @@ def end(self): if value: trek_parent_instance = Trek.objects.filter(eid=key) if not trek_parent_instance: + self.add_warning(_(f"Trying to retrieve children for missing trek : could not find trek with UUID {key}")) return order = 0 for child in value: diff --git a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json index 06acc25869..ed743a54b3 100644 --- a/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json +++ b/geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json @@ -29,6 +29,14 @@ { "uuid": "b2aea892-5e6e-4daa-a750-7d2ee52d3fe1", "steps": [] + }, + { + "uuid": "b2aea666-5e6e-4daa-a750-7d2ee52d3fe1", + "steps": [ + { + "uuid": "c9567576-2934-43ab-979e-e13d02c671a9" + } + ] } ] } diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index e15f54c2e2..4080c78ec6 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -375,6 +375,7 @@ def test_children_do_not_exist(self, mocked_head, mocked_get): call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2, stdout=output) self.assertIn("One trek has not be generated for Boucle du Pic des Trois Seigneurs : could not find trek with UUID c9567576-2934-43ab-666e-e13d02c671a9,\n", output.getvalue()) + self.assertIn("Trying to retrieve children for missing trek : could not find trek with UUID b2aea666-5e6e-4daa-a750-7d2ee52d3fe1", output.getvalue()) @mock.patch('requests.get') @mock.patch('requests.head')