From 4fe0ebfd3081d934284977015de10429496649e8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 16 Jan 2023 19:31:16 +0100 Subject: [PATCH 1/6] fix: Fix apidae parser should not update every time --- docs/changelog.rst | 1 + geotrek/common/parsers.py | 7 +++++-- geotrek/trekking/parsers.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e08d92b693..f3766406ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -39,6 +39,7 @@ In preparation for HD Views developments (PR #3298) - Modify site's geometry before saving to avoid edition and export of shapefiles (#3399) - Fix API V2 cache key with X-Forwarded-Proto header (#3404) - Check pictogram exist on categories during generation of pdfs +- Fix ApidaeParsers does not update every time 2.94.0 (2022-12-12) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index b8ef7f7b94..be9515eddd 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -252,14 +252,17 @@ def parse_translation_field(self, dst, src, val): """ val = val or "" modified = False - + old_values = {} + for lang in settings.MODELTRANSLATION_LANGUAGES: + dst_field_lang = '{field}_{lang}'.format(field=dst, lang=lang) + old_values[lang] = getattr(self.obj, dst_field_lang) if hasattr(self, 'filter_{0}'.format(dst)): val = getattr(self, 'filter_{0}'.format(dst))(src, val) else: val = self.apply_filter(dst, src, val) for lang in settings.MODELTRANSLATION_LANGUAGES: dst_field_lang = '{field}_{lang}'.format(field=dst, lang=lang) - old = getattr(self.obj, dst_field_lang) + old = old_values[lang] # Field not translated, use same val for all translated val = val or "" diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 56266ef46a..974940a3e5 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -1007,9 +1007,9 @@ def _make_duration(duration_in_minutes=None, duration_in_days=None): """Returns the duration in hours. The method expects one argument or the other, not both. If both arguments have non-zero values the method only considers `duration_in_minutes` and discards `duration_in_days`.""" if duration_in_minutes: - return (Decimal(duration_in_minutes) / Decimal(60)).quantize(Decimal('.01')) + return float((Decimal(duration_in_minutes) / Decimal(60)).quantize(Decimal('.01'))) elif duration_in_days: - return duration_in_days * 24 + return float(duration_in_days * 24) else: return None From adc1eaa65651bc9f18e471a3e96314b7d2ed8174 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 17 Jan 2023 14:52:49 +0100 Subject: [PATCH 2/6] fix: check if elment from apidae has a plan --- geotrek/trekking/parsers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/geotrek/trekking/parsers.py b/geotrek/trekking/parsers.py index 974940a3e5..0e93821ca6 100644 --- a/geotrek/trekking/parsers.py +++ b/geotrek/trekking/parsers.py @@ -869,6 +869,8 @@ def _find_gpx_plan_in_multimedia_items(items): plans = list(filter(lambda item: item['type'] == 'PLAN', items)) if len(plans) > 1: raise RowImportError("APIDAE Trek has more than one map defined") + if len(plans) == 0: + raise RowImportError("APIDAE Trek has no map defined") return plans[0] @staticmethod From 26d2b7116520a0413806a34919d39348c6b54693 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 17 Jan 2023 11:26:33 +0100 Subject: [PATCH 3/6] fix: improve parse_translation_field parsers --- geotrek/common/parsers.py | 21 +++++++++++---------- geotrek/trekking/tests/test_parsers.py | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index be9515eddd..e51f0c9e88 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -243,7 +243,6 @@ def parse_real_field(self, dst, src, val): def parse_translation_field(self, dst, src, val): """Specific treatment for translated fields TODO: check self.default_language to get default values - TODO: handle flow with a field for each language (ex: APIDAE) TODO: compare each translated fields with source fields : this only compares old 'name' with new 'name' but it should compare - old 'name_en' with new 'name_en', @@ -257,19 +256,21 @@ def parse_translation_field(self, dst, src, val): dst_field_lang = '{field}_{lang}'.format(field=dst, lang=lang) old_values[lang] = getattr(self.obj, dst_field_lang) if hasattr(self, 'filter_{0}'.format(dst)): - val = getattr(self, 'filter_{0}'.format(dst))(src, val) + val_default_language = getattr(self, 'filter_{0}'.format(dst))(src, val) else: - val = self.apply_filter(dst, src, val) + val_default_language = self.apply_filter(dst, src, val) + for lang in settings.MODELTRANSLATION_LANGUAGES: dst_field_lang = '{field}_{lang}'.format(field=dst, lang=lang) - old = old_values[lang] + new_value = getattr(self.obj, dst_field_lang) + old_value = old_values[lang] # Field not translated, use same val for all translated - val = val or "" - - if old != val: - # Set dst_field_lang only if empty - if not old: - self.set_value(dst_field_lang, src, val) + val_default_language = val_default_language or "" + if not new_value: + # If there is no new value check if old value is different form the default value + # Set dst_field_lang only if new empty + if old_value != val_default_language: + self.set_value(dst_field_lang, src, val_default_language) modified = True return modified diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 0c6e7a52b3..76b4cc0df8 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -1191,7 +1191,7 @@ def test_giving_both_duration_arguments_only_duration_in_minutes_is_considered(s self.assertAlmostEqual(ApidaeTrekParser._make_duration(duration_in_minutes=90, duration_in_days=0.5), 1.5) def test_it_rounds_output_to_two_decimal_places(self): - self.assertEqual(Decimal(ApidaeTrekParser._make_duration(duration_in_minutes=20)), Decimal('0.33')) + self.assertEqual(ApidaeTrekParser._make_duration(duration_in_minutes=20), 0.33) class TestApidaePOIParser(ApidaePOIParser): From f61b5c1f84ae2cdffe2b16c24b578c0629e41837 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 17 Jan 2023 11:32:49 +0100 Subject: [PATCH 4/6] fix: linting --- geotrek/trekking/tests/test_parsers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index 76b4cc0df8..b3df2a2ae2 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -1,5 +1,4 @@ from datetime import date -from decimal import Decimal import json import os from copy import copy From 5ea8e178dceeb542ab4c16b3b4d0b79f58ab0052 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 17 Jan 2023 13:32:34 +0100 Subject: [PATCH 5/6] test: add test no plan --- .../data/apidae_trek_parser/trek_no_plan_error.json | 12 ++++++++++++ geotrek/trekking/tests/test_parsers.py | 11 +++++++++++ 2 files changed, 23 insertions(+) create mode 100644 geotrek/trekking/tests/data/apidae_trek_parser/trek_no_plan_error.json diff --git a/geotrek/trekking/tests/data/apidae_trek_parser/trek_no_plan_error.json b/geotrek/trekking/tests/data/apidae_trek_parser/trek_no_plan_error.json new file mode 100644 index 0000000000..81daf1caa4 --- /dev/null +++ b/geotrek/trekking/tests/data/apidae_trek_parser/trek_no_plan_error.json @@ -0,0 +1,12 @@ +{ + "numFound": 1, + "objetsTouristiques": [ + { + "id": 123123, + "multimedias": [], + "nom": { + "libelleFr": "Une belle randonnée de test sans plan..." + } + } + ] +} \ No newline at end of file diff --git a/geotrek/trekking/tests/test_parsers.py b/geotrek/trekking/tests/test_parsers.py index b3df2a2ae2..42ce41b782 100644 --- a/geotrek/trekking/tests/test_parsers.py +++ b/geotrek/trekking/tests/test_parsers.py @@ -817,6 +817,17 @@ def test_trek_not_imported_when_no_gpx_file(self, mocked_get): self.assertEqual(Trek.objects.count(), 0) self.assertIn('pas au format GPX', output_stdout.getvalue()) + @mock.patch('requests.get') + def test_trek_not_imported_when_no_plan(self, mocked_get): + output_stdout = StringIO() + mocked_get.side_effect = self.make_dummy_get('trek_no_plan_error.json') + + call_command('import', 'geotrek.trekking.tests.test_parsers.TestApidaeTrekParser', verbosity=2, + stdout=output_stdout) + + self.assertEqual(Trek.objects.count(), 0) + self.assertIn('APIDAE Trek has no map defined', output_stdout.getvalue()) + @mock.patch('requests.get') def test_trek_linked_entities_are_imported(self, mocked_get): mocked_get.side_effect = self.make_dummy_get('a_trek.json') From 43530210384908121cf53db6bb06791fbdb6d904 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Tue, 17 Jan 2023 14:49:14 +0100 Subject: [PATCH 6/6] doc: add comment parsers translation field --- geotrek/common/parsers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index e51f0c9e88..7b4b7f5588 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -252,9 +252,12 @@ def parse_translation_field(self, dst, src, val): val = val or "" modified = False old_values = {} + # We keep every old values for each langs to get tracability during filter or apply filter for lang in settings.MODELTRANSLATION_LANGUAGES: dst_field_lang = '{field}_{lang}'.format(field=dst, lang=lang) old_values[lang] = getattr(self.obj, dst_field_lang) + # If during filter, the traduction of the field has been changed + # we can still check if this value has been changed if hasattr(self, 'filter_{0}'.format(dst)): val_default_language = getattr(self, 'filter_{0}'.format(dst))(src, val) else: @@ -266,9 +269,10 @@ def parse_translation_field(self, dst, src, val): old_value = old_values[lang] # Field not translated, use same val for all translated val_default_language = val_default_language or "" + # Set val_default_language only if new empty if not new_value: # If there is no new value check if old value is different form the default value - # Set dst_field_lang only if new empty + # If this is the same, it means old value was empty and was fill with default value in previous import if old_value != val_default_language: self.set_value(dst_field_lang, src, val_default_language) modified = True