From 07b9331990173b58207dccc0311468e1ee910136 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Thu, 5 Jan 2023 18:39:12 +0100 Subject: [PATCH 1/8] :fix: Force simple geometrycollection on sites --- docs/changelog.rst | 2 ++ geotrek/outdoor/models.py | 11 +++++++++++ geotrek/outdoor/tests/test_models.py | 12 +++++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ad07aa3916..1ba2d1d548 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,8 @@ In preparation for HD Views developments (PR #3298) **Bug fixes** - Recreate cache folders if missing. (#3384) +- Modify site's geometry before saving to avoid edition and export of shapefiles (#3399) + 2.94.0 (2022-12-12) ----------------------- diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index 776a511137..a1d73cc501 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -2,10 +2,12 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models +from django.contrib.gis.geos import GEOSGeometry, GeometryCollection from django.contrib.gis.measure import D from django.contrib.postgres.indexes import GistIndex from django.core.validators import MinValueValidator from django.db.models import Q +from django.db import connection from django.utils.html import escape from django.utils.translation import gettext_lazy as _ from mptt.models import MPTTModel, TreeForeignKey @@ -299,6 +301,15 @@ def site_interventions(self): qs |= Q(target_id__in=topologies) & ~Q(target_type__in=not_topology_content_types) return Intervention.objects.existing().filter(qs).distinct('pk') + def save(self, *args, **kwargs): + with connection.cursor() as c: + c.callproc('ST_Dump', [self.geom.wkt]) + geometries = [] + for ids, geometry in c.fetchall(): + geometries.append(GEOSGeometry(geometry, srid=settings.SRID)) + self.geom = GeometryCollection(geometries, srid=settings.SRID) + return super().save(*args, **kwargs) + Path.add_property('sites', lambda self: intersecting(Site, self), _("Sites")) Topology.add_property('sites', lambda self: intersecting(Site, self), _("Sites")) diff --git a/geotrek/outdoor/tests/test_models.py b/geotrek/outdoor/tests/test_models.py index eeaf69ab8a..48288a7aa7 100644 --- a/geotrek/outdoor/tests/test_models.py +++ b/geotrek/outdoor/tests/test_models.py @@ -2,7 +2,7 @@ from django.contrib.gis.geos import Polygon from django.contrib.gis.geos.collections import GeometryCollection -from django.contrib.gis.geos.point import Point +from django.contrib.gis.geos.point import Point, GEOSGeometry from django.test import TestCase, override_settings from geotrek.common.tests.factories import OrganismFactory @@ -25,6 +25,16 @@ def test_published_children_by_lang(self): SiteFactory(name='child3', parent=parent, published_fr=True) self.assertQuerysetEqual(parent.published_children, ['', '']) + def test_validate_collection_geometrycollection(self): + site_simple = SiteFactory.create(name='site', description='LUL', geom='GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1))))') + self.assertEqual(site_simple.geom.wkt, + GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1)))').wkt + ) + site_complex_geom = SiteFactory.create(name='site', geom='GEOMETRYCOLLECTION(MULTIPOINT(0 0, 1 1), POLYGON((1 1, 2 2, 1 2, 1 1))))') + self.assertEqual(site_complex_geom.geom.wkt, + GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POLYGON((1 1, 2 2, 1 2, 1 1)))').wkt + ) + class SiteSuperTest(TestCase): @classmethod From 41f460692e284e102167a1c715329e37c9813a12 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 13:37:08 +0100 Subject: [PATCH 2/8] :feat: Add geometrycollection flattening sites/outdoors trigger --- geotrek/outdoor/models.py | 9 ------ .../sql/post_30_geometrycollections.sql | 30 +++++++++++++++++++ .../templates/outdoor/sql/pre_10_cleanup.sql | 1 + geotrek/outdoor/tests/test_models.py | 23 ++++++++++++-- 4 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index a1d73cc501..3b130d5fce 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -301,15 +301,6 @@ def site_interventions(self): qs |= Q(target_id__in=topologies) & ~Q(target_type__in=not_topology_content_types) return Intervention.objects.existing().filter(qs).distinct('pk') - def save(self, *args, **kwargs): - with connection.cursor() as c: - c.callproc('ST_Dump', [self.geom.wkt]) - geometries = [] - for ids, geometry in c.fetchall(): - geometries.append(GEOSGeometry(geometry, srid=settings.SRID)) - self.geom = GeometryCollection(geometries, srid=settings.SRID) - return super().save(*args, **kwargs) - Path.add_property('sites', lambda self: intersecting(Site, self), _("Sites")) Topology.add_property('sites', lambda self: intersecting(Site, self), _("Sites")) diff --git a/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql b/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql new file mode 100644 index 0000000000..62959fab6e --- /dev/null +++ b/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql @@ -0,0 +1,30 @@ +------------------------------------------------------------------------------- +-- Compute elevation and elevation-based indicators +------------------------------------------------------------------------------- + +-- Possible to have collection of GeometryCollection +-- we need to flatten the geometrycollection to avoid problem with export and edition (#3397) + + +CREATE FUNCTION {{ schema_geotrek }}.geometrycollection_flatten_outdoor_iu() RETURNS trigger SECURITY DEFINER AS $$ +DECLARE + geoms geometry[]; + geom geometry; +BEGIN + FOR geom IN SELECT (ST_Dump(NEW.geom)).geom LOOP + -- Update site geometry + geoms := array_append(geoms, geom); + END LOOP; + NEW.geom := ST_ForceCollection(ST_COLLECT(geoms)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + + +CREATE TRIGGER outdoor_site_30_geometrycollection_flatten_iu_tgr +BEFORE INSERT OR UPDATE OF geom ON outdoor_site +FOR EACH ROW EXECUTE PROCEDURE geometrycollection_flatten_outdoor_iu(); + +CREATE TRIGGER outdoor_course_30_geometrycollection_flatten_iu_tgr +BEFORE INSERT OR UPDATE OF geom ON outdoor_course +FOR EACH ROW EXECUTE PROCEDURE geometrycollection_flatten_outdoor_iu(); diff --git a/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql b/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql index f3ee9de922..ea451d3dc6 100644 --- a/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql +++ b/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql @@ -1,6 +1,7 @@ --10 DROP FUNCTION IF EXISTS elevation_outdoor_iu() CASCADE; +DROP FUNCTION IF EXISTS geometrycollection_flatten_outdoor_iu() CASCADE; --20 diff --git a/geotrek/outdoor/tests/test_models.py b/geotrek/outdoor/tests/test_models.py index 48288a7aa7..e3580b7971 100644 --- a/geotrek/outdoor/tests/test_models.py +++ b/geotrek/outdoor/tests/test_models.py @@ -26,14 +26,33 @@ def test_published_children_by_lang(self): self.assertQuerysetEqual(parent.published_children, ['', '']) def test_validate_collection_geometrycollection(self): - site_simple = SiteFactory.create(name='site', description='LUL', geom='GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1))))') + site_simple = SiteFactory.create(name='site', + geom='GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1))))') + site_simple.refresh_from_db() self.assertEqual(site_simple.geom.wkt, GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1)))').wkt ) - site_complex_geom = SiteFactory.create(name='site', geom='GEOMETRYCOLLECTION(MULTIPOINT(0 0, 1 1), POLYGON((1 1, 2 2, 1 2, 1 1))))') + site_complex_geom = SiteFactory.create(name='site', + geom='GEOMETRYCOLLECTION(MULTIPOINT(0 0, 1 1), ' + 'POLYGON((1 1, 2 2, 1 2, 1 1))))') + site_complex_geom.refresh_from_db() self.assertEqual(site_complex_geom.geom.wkt, GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POLYGON((1 1, 2 2, 1 2, 1 1)))').wkt ) + site_multiple_point = SiteFactory.create(name='site', + geom='GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POINT(1 2))') + site_multiple_point.refresh_from_db() + self.assertEqual(site_multiple_point.geom.wkt, + GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POINT(1 2)))').wkt + ) + site_multiple_geomcollection = SiteFactory.create(name='site', + geom='GEOMETRYCOLLECTION(' + 'GEOMETRYCOLLECTION(POINT(0 0)),' + 'GEOMETRYCOLLECTION(POINT(1 1)), ' + 'GEOMETRYCOLLECTION(POINT(1 2)))') + site_multiple_geomcollection.refresh_from_db() + self.assertEqual(site_multiple_geomcollection.geom.wkt, + 'GEOMETRYCOLLECTION (POINT (0 0), POINT (1 1), POINT (1 2))') class SiteSuperTest(TestCase): From ffd332d20cc94ffafceaae99042033bab4d371c8 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 13:37:35 +0100 Subject: [PATCH 3/8] :feat: Add migration flettening geomtrycollection --- .../0042_flatten_geometrycollection.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 geotrek/outdoor/migrations/0042_flatten_geometrycollection.py diff --git a/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py b/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py new file mode 100644 index 0000000000..df8fc1a86d --- /dev/null +++ b/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1.7 on 2021-03-15 15:12 + +from django.db import migrations + + +def flatten_geometrycollection(apps, schema_editor): + Site = apps.get_model('outdoor', 'Site') + Course = apps.get_model('outdoor', 'Course') + Site.objects.bulk_update(Site.objects.all(), ['geom']) + Course.objects.bulk_update(Course.objects.all(), ['geom']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('outdoor', '0041_auto_20221110_1128'), + ] + + operations = [ + migrations.RunPython(flatten_geometrycollection), + ] From fd50aaef655fed1be3cdc5688ff62b5872bc25b0 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 13:45:48 +0100 Subject: [PATCH 4/8] :feat: Remove model import save --- geotrek/outdoor/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index 3b130d5fce..776a511137 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -2,12 +2,10 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models -from django.contrib.gis.geos import GEOSGeometry, GeometryCollection from django.contrib.gis.measure import D from django.contrib.postgres.indexes import GistIndex from django.core.validators import MinValueValidator from django.db.models import Q -from django.db import connection from django.utils.html import escape from django.utils.translation import gettext_lazy as _ from mptt.models import MPTTModel, TreeForeignKey From c177a2c48cee441412e0a53fafdb4bf91adba3aa Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 14:28:05 +0100 Subject: [PATCH 5/8] :feat: Move function flatten geometry collection in utils common --- .../common/sql/post_10_utilities.sql | 16 ++++++++++++++ .../sql/post_30_geometrycollections.sql | 22 ++----------------- .../templates/outdoor/sql/pre_10_cleanup.sql | 2 +- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/geotrek/common/templates/common/sql/post_10_utilities.sql b/geotrek/common/templates/common/sql/post_10_utilities.sql index 7c60e80f2f..806474efa8 100644 --- a/geotrek/common/templates/common/sql/post_10_utilities.sql +++ b/geotrek/common/templates/common/sql/post_10_utilities.sql @@ -22,3 +22,19 @@ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql; + +-- Possible to have collection of GeometryCollection +-- we need to flatten the geometrycollection to avoid problem with export and edition (#3397) + +CREATE FUNCTION {{ schema_geotrek }}.flatten_geometrycollection_iu() RETURNS trigger SECURITY DEFINER AS $$ +DECLARE + geoms geometry[]; + geom geometry; +BEGIN + FOR geom IN SELECT (ST_Dump(NEW.geom)).geom LOOP + geoms := array_append(geoms, geom); + END LOOP; + NEW.geom := ST_ForceCollection(ST_COLLECT(geoms)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql b/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql index 62959fab6e..72c9a43393 100644 --- a/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql +++ b/geotrek/outdoor/templates/outdoor/sql/post_30_geometrycollections.sql @@ -2,29 +2,11 @@ -- Compute elevation and elevation-based indicators ------------------------------------------------------------------------------- --- Possible to have collection of GeometryCollection --- we need to flatten the geometrycollection to avoid problem with export and edition (#3397) - - -CREATE FUNCTION {{ schema_geotrek }}.geometrycollection_flatten_outdoor_iu() RETURNS trigger SECURITY DEFINER AS $$ -DECLARE - geoms geometry[]; - geom geometry; -BEGIN - FOR geom IN SELECT (ST_Dump(NEW.geom)).geom LOOP - -- Update site geometry - geoms := array_append(geoms, geom); - END LOOP; - NEW.geom := ST_ForceCollection(ST_COLLECT(geoms)); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - CREATE TRIGGER outdoor_site_30_geometrycollection_flatten_iu_tgr BEFORE INSERT OR UPDATE OF geom ON outdoor_site -FOR EACH ROW EXECUTE PROCEDURE geometrycollection_flatten_outdoor_iu(); +FOR EACH ROW EXECUTE PROCEDURE flatten_geometrycollection_iu(); CREATE TRIGGER outdoor_course_30_geometrycollection_flatten_iu_tgr BEFORE INSERT OR UPDATE OF geom ON outdoor_course -FOR EACH ROW EXECUTE PROCEDURE geometrycollection_flatten_outdoor_iu(); +FOR EACH ROW EXECUTE PROCEDURE flatten_geometrycollection_iu(); diff --git a/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql b/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql index ea451d3dc6..4444c56d7b 100644 --- a/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql +++ b/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql @@ -1,7 +1,7 @@ --10 DROP FUNCTION IF EXISTS elevation_outdoor_iu() CASCADE; -DROP FUNCTION IF EXISTS geometrycollection_flatten_outdoor_iu() CASCADE; +DROP FUNCTION IF EXISTS flatten_geometrycollection_iu() CASCADE; --20 From e580332ff7dbe58502ee905256f57b2e5d9de4a5 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 14:29:42 +0100 Subject: [PATCH 6/8] :feat: add reverse migration flatten geometrycollection --- geotrek/outdoor/migrations/0042_flatten_geometrycollection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py b/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py index df8fc1a86d..507c80c7f0 100644 --- a/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py +++ b/geotrek/outdoor/migrations/0042_flatten_geometrycollection.py @@ -17,5 +17,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(flatten_geometrycollection), + migrations.RunPython(flatten_geometrycollection, migrations.RunPython.noop), ] From cfdd97e42540517986ff8d27e639686d1cee9ab3 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 14:30:13 +0100 Subject: [PATCH 7/8] :fix: add refresh from db directly in model save --- geotrek/outdoor/models.py | 4 ++++ geotrek/outdoor/tests/test_models.py | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/geotrek/outdoor/models.py b/geotrek/outdoor/models.py index 776a511137..91b50a8344 100644 --- a/geotrek/outdoor/models.py +++ b/geotrek/outdoor/models.py @@ -299,6 +299,10 @@ def site_interventions(self): qs |= Q(target_id__in=topologies) & ~Q(target_type__in=not_topology_content_types) return Intervention.objects.existing().filter(qs).distinct('pk') + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + self.refresh_from_db() + Path.add_property('sites', lambda self: intersecting(Site, self), _("Sites")) Topology.add_property('sites', lambda self: intersecting(Site, self), _("Sites")) diff --git a/geotrek/outdoor/tests/test_models.py b/geotrek/outdoor/tests/test_models.py index e3580b7971..a5f1dcb417 100644 --- a/geotrek/outdoor/tests/test_models.py +++ b/geotrek/outdoor/tests/test_models.py @@ -28,20 +28,17 @@ def test_published_children_by_lang(self): def test_validate_collection_geometrycollection(self): site_simple = SiteFactory.create(name='site', geom='GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1))))') - site_simple.refresh_from_db() self.assertEqual(site_simple.geom.wkt, GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1)))').wkt ) site_complex_geom = SiteFactory.create(name='site', geom='GEOMETRYCOLLECTION(MULTIPOINT(0 0, 1 1), ' 'POLYGON((1 1, 2 2, 1 2, 1 1))))') - site_complex_geom.refresh_from_db() self.assertEqual(site_complex_geom.geom.wkt, GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POLYGON((1 1, 2 2, 1 2, 1 1)))').wkt ) site_multiple_point = SiteFactory.create(name='site', geom='GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POINT(1 2))') - site_multiple_point.refresh_from_db() self.assertEqual(site_multiple_point.geom.wkt, GEOSGeometry('GEOMETRYCOLLECTION(POINT(0 0), POINT(1 1), POINT(1 2)))').wkt ) @@ -50,7 +47,6 @@ def test_validate_collection_geometrycollection(self): 'GEOMETRYCOLLECTION(POINT(0 0)),' 'GEOMETRYCOLLECTION(POINT(1 1)), ' 'GEOMETRYCOLLECTION(POINT(1 2)))') - site_multiple_geomcollection.refresh_from_db() self.assertEqual(site_multiple_geomcollection.geom.wkt, 'GEOMETRYCOLLECTION (POINT (0 0), POINT (1 1), POINT (1 2))') From 5186053efc78ec93b33307053e8653f3e5ee6545 Mon Sep 17 00:00:00 2001 From: LePetitTim Date: Mon, 9 Jan 2023 16:43:29 +0100 Subject: [PATCH 8/8] :fix: Move clean procedure flatten geometrycollection --- geotrek/common/templates/common/sql/pre_10_cleanup.sql | 1 + geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/geotrek/common/templates/common/sql/pre_10_cleanup.sql b/geotrek/common/templates/common/sql/pre_10_cleanup.sql index 791fbd08e8..9d47546a72 100644 --- a/geotrek/common/templates/common/sql/pre_10_cleanup.sql +++ b/geotrek/common/templates/common/sql/pre_10_cleanup.sql @@ -3,3 +3,4 @@ DROP TABLE IF EXISTS south_migrationhistory; -- legacy, replaced by django_migr DROP FUNCTION IF EXISTS ft_date_insert() CASCADE; DROP FUNCTION IF EXISTS ft_date_update() CASCADE; DROP FUNCTION IF EXISTS ft_uuid_insert() CASCADE; +DROP FUNCTION IF EXISTS flatten_geometrycollection_iu() CASCADE; diff --git a/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql b/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql index 4444c56d7b..f3ee9de922 100644 --- a/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql +++ b/geotrek/outdoor/templates/outdoor/sql/pre_10_cleanup.sql @@ -1,7 +1,6 @@ --10 DROP FUNCTION IF EXISTS elevation_outdoor_iu() CASCADE; -DROP FUNCTION IF EXISTS flatten_geometrycollection_iu() CASCADE; --20