[Fixes #8905] Implement API and remote file path support for layer da…

…ta replace (#9062) * [Fixes #8905] Implement replace layer * [Fixes #8905] Rollback test xml file * [Fixes #8905] Add APIv2 api for layer replace and add test coverage * [Fixes #8905] Add APIv2 api for layer replace and add test coverage * [Fixes #8905] fix check uuid on overwrite * [Fixes #8905] Add check for data consistency on replace/append layer * [Fixes #8905] Add check for data consistency on replace/append layer * [Fixes #8905] Fix broken tests and add unzip for zip replaces * [Fixes #8905] Test fix broken test for replace integration * [Fixes #8905] flake8 fix * [Fixes #8905] Test fix broken test for replace integration * [Fixes #8905] fix test * [Fixes #8905] fix test * [ISSUE #8905] Add csrf exception for upload view, add missing info * [ISSUE #8905] Fix upload flow * [ISSUE #8905] Fix upload flow * [ISSUE #8905] Fix upload flow * [Fixes #8905] Fix build for replace layer * [Fixes #8905] Rollback changes on env_dev * [Fixes #8905] Fix broken tests * [Fixes #9064Improve Upload Workflow resources state management] * [Fixes #9064] Improve Upload Workflow resources state management * [Fixes #8905] Remove unwanted code * [Fixes #8905] Remove unwanted code * [Fixes #8905] Fix broken tests * [Fixes #8905] Fix broken tests * [Fixes #8905] Fix broken tests * - Missing headers Co-authored-by: afabiani <[email protected]>
GeoNode · Apr 11, 2022 · e66da3b · e66da3b
1 parent 23c5db3
commit e66da3b
Show file tree

Hide file tree

Showing 9 changed files with 306 additions and 101 deletions.
diff --git a/geonode/layers/api/exceptions.py b/geonode/layers/api/exceptions.py
@@ -0,0 +1,25 @@
+#########################################################################
+#
+# Copyright (C) 2022 OSGeo
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+from rest_framework.exceptions import APIException
+
+
+class LayerReplaceException(APIException):
+    status_code = 500
+    default_detail = 'Error during layer replace.'
+    default_code = 'layer_replace_exception'
diff --git a/geonode/layers/api/serializers.py b/geonode/layers/api/serializers.py
@@ -86,7 +86,7 @@ class Meta:
             'is_mosaic', 'has_time', 'has_elevation', 'time_regex', 'elevation_regex',
             'use_featureinfo_custom_template', 'featureinfo_custom_template',
             'default_style', 'styles', 'attribute_set',
-            'ptype', 'ows_url'
+            'ptype', 'ows_url', 'upload_session'
         )
 
     name = serializers.CharField(read_only=True)

diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py
@@ -17,15 +17,21 @@
 #
 #########################################################################
 import logging
+import shutil
+import tempfile
+from unittest.mock import patch
+from django.conf import settings
 
-from django.urls import reverse
+import gisdata
 from django.conf.urls import url
-from rest_framework.test import APITestCase, URLPatternsTestCase
-
+from django.contrib.auth import get_user_model
+from django.urls import reverse
 from geonode import geoserver
+from geonode.base.populate_test_data import create_models
+from geonode.geoserver.createlayer.utils import create_layer
 from geonode.layers.models import Layer
 from geonode.utils import check_ogc_backend
-from geonode.base.populate_test_data import create_models
+from rest_framework.test import APITestCase, URLPatternsTestCase
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +60,9 @@ def setUp(self):
         create_models(b'map')
         create_models(b'layer')
 
+    def tearDown(self) -> None:
+        Layer.objects.filter(name='new_name').delete()
+
     def test_layers(self):
         """
         Ensure we can access the Layers list.
@@ -96,3 +105,95 @@ def test_raw_HTML_stripped_properties(self):
         self.assertEqual(response.data['layer']['raw_constraints_other'], "None")
         self.assertEqual(response.data['layer']['raw_supplemental_information'], "No information provided í £682m")
         self.assertEqual(response.data['layer']['raw_data_quality_statement'], "OK    1 2   a b")
+
+    def test_layer_replace_anonymous_should_raise_error(self):
+        layer = Layer.objects.first()
+        url = reverse("layers-replace-layer", args=(layer.id,))
+
+        expected = {"detail": "Authentication credentials were not provided."}
+
+        response = self.client.put(url)
+        self.assertEqual(403, response.status_code)
+        self.assertDictEqual(expected, response.json())
+
+    def test_layer_replace_should_redirect_for_not_accepted_method(self):
+        layer = Layer.objects.first()
+        url = reverse("layers-replace-layer", args=(layer.id,))
+        self.client.login(username="admin", password="admin")
+
+        response = self.client.post(url)
+        self.assertEqual(405, response.status_code)
+
+        response = self.client.get(url)
+        self.assertEqual(405, response.status_code)
+
+        response = self.client.patch(url)
+        self.assertEqual(405, response.status_code)
+
+    def test_layer_replace_should_raise_error_if_layer_does_not_exists(self):
+        url = reverse("layers-replace-layer", args=(999999999999999,))
+
+        expected = {"detail": "Layer with ID 999999999999999 is not available"}
+
+        self.client.login(username="admin", password="admin")
+
+        response = self.client.put(url)
+        self.assertEqual(404, response.status_code)
+        self.assertDictEqual(expected, response.json())
+
+    @patch("geonode.layers.views.validate_input_source")
+    def test_layer_replace_should_work(self, _validate_input_source):
+
+        _validate_input_source.return_value = True
+
+        admin = get_user_model().objects.get(username='admin')
+
+        layer = create_layer(
+            "new_name",
+            "new_name",
+            admin,
+            "Point",
+        )
+        cnt = Layer.objects.count()
+
+        self.assertEqual('No abstract provided', layer.abstract)
+        self.assertEqual(Layer.objects.count(), cnt)
+
+        layer.refresh_from_db()
+        logger.error(layer.alternate)
+        # renaming the file in the same way as the layer name
+        # the filename must be consistent with the layer name
+        tempdir = tempfile.mkdtemp(dir=settings.STATIC_ROOT)
+
+        shutil.copyfile(f"{gisdata.GOOD_DATA}/vector/single_point.shp", f"{tempdir}/{layer.alternate.split(':')[1]}.shp")
+        shutil.copyfile(f"{gisdata.GOOD_DATA}/vector/single_point.dbf", f"{tempdir}/{layer.alternate.split(':')[1]}.dbf")
+        shutil.copyfile(f"{gisdata.GOOD_DATA}/vector/single_point.prj", f"{tempdir}/{layer.alternate.split(':')[1]}.prj")
+        shutil.copyfile(f"{gisdata.GOOD_DATA}/vector/single_point.shx", f"{tempdir}/{layer.alternate.split(':')[1]}.shx")
+        shutil.copyfile(f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml", f"{tempdir}/{layer.alternate.split(':')[1]}.xml")
+
+        payload = {
+            "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}',
+            "time": "false",
+            "charset": "UTF-8",
+            "store_spatial_files": False,
+            "base_file_path": f"{tempdir}/{layer.alternate.split(':')[1]}.shp",
+            "dbf_file_path": f"{tempdir}/{layer.alternate.split(':')[1]}.dbf",
+            "prj_file_path": f"{tempdir}/{layer.alternate.split(':')[1]}.prj",
+            "shx_file_path": f"{tempdir}/{layer.alternate.split(':')[1]}.shx",
+            "xml_file_path": f"{tempdir}/{layer.alternate.split(':')[1]}.xml"
+        }
+
+        url = reverse("layers-replace-layer", args=(layer.id,))
+
+        self.client.login(username="admin", password="admin")
+
+        response = self.client.put(url, data=payload)
+        self.assertEqual(200, response.status_code)
+
+        layer.refresh_from_db()
+        # evaluate that the abstract is updated and the number of available layer is not changed
+        self.assertEqual(Layer.objects.count(), cnt)
+        self.assertEqual('real abstract', layer.abstract)
+
+        if tempdir:
+            shutil.rmtree(tempdir)
diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py
@@ -19,13 +19,18 @@
 from dynamic_rest.viewsets import DynamicModelViewSet
 from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter
 
+from rest_framework.exceptions import AuthenticationFailed, NotFound
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.authentication import SessionAuthentication, BasicAuthentication
 from oauth2_provider.contrib.rest_framework import OAuth2Authentication
 from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter
 from geonode.base.api.permissions import IsOwnerOrReadOnly
 from geonode.base.api.pagination import GeoNodeApiPagination
+from geonode.layers.api.exceptions import LayerReplaceException
 from geonode.layers.models import Layer
+from drf_spectacular.utils import extend_schema
+from rest_framework.decorators import action
+from geonode.layers.views import layer_replace
 
 from .serializers import LayerSerializer
 from .permissions import LayerPermissionsFilter
@@ -49,3 +54,31 @@ class LayerViewSet(DynamicModelViewSet):
     queryset = Layer.objects.all().order_by('-last_updated')
     serializer_class = LayerSerializer
     pagination_class = GeoNodeApiPagination
+
+    @extend_schema(
+        methods=["put"],
+        responses={200},
+        description="API endpoint allowing to replace a layer."
+    )
+    @action(
+        detail=False,
+        url_path="(?P<layer_id>\d+)/replace",  # noqa
+        url_name="replace-layer",
+        methods=["put"]
+    )
+    def replace(self, request, layer_id=None):
+        user = request.user
+        if not user or not user.is_authenticated:
+            raise AuthenticationFailed
+
+        if not self.queryset.filter(id=layer_id).exists():
+            raise NotFound(detail=f"Layer with ID {layer_id} is not available")
+
+        alternate = self.queryset.get(id=layer_id).alternate
+
+        response = layer_replace(request=request, layername=alternate)
+
+        if response.status_code != 200:
+            raise LayerReplaceException(detail=response.content)
+
+        return response
diff --git a/geonode/layers/models.py b/geonode/layers/models.py
@@ -286,7 +286,7 @@ def gtype(self):
             _attrs = Attribute.objects.filter(layer=self)
         if _attrs.filter(attribute='the_geom').exists():
             _att_type = _attrs.filter(attribute='the_geom').first().attribute_type
-            _gtype = re.match(r'\(\'gml:(.*?)\',', _att_type)
+            _gtype = re.match(r'gml:(.*)PropertyType', _att_type)
             return _gtype.group(1) if _gtype else None
         return None
 

diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py
@@ -1115,61 +1115,68 @@ def validate_input_source(layer, filename, files, gtype=None, action_type='repla
         raise Exception(_(
             f"You are attempting to {action_type} a raster layer with a vector."))
 
-    if layer.is_vector():
+    if not layer.is_vector():
+        return True
+    absolute_base_file = None
+    try:
+        if 'shp' in files and os.path.exists(files['shp']):
+            absolute_base_file = (
+                _fixup_base_file(files['shp'])
+                if not action_type == 'replace'
+                else files['shp']
+            )
+        elif 'zip' in files and os.path.exists(files['zip']):
+            absolute_base_file = (
+                _fixup_base_file(files['zip'])
+            )
+    except Exception:
         absolute_base_file = None
-        try:
-            if 'shp' in files and os.path.exists(files['shp']):
-                absolute_base_file = _fixup_base_file(files['shp'])
-            elif 'zip' in files and os.path.exists(files['zip']):
-                absolute_base_file = _fixup_base_file(files['zip'])
-        except Exception:
-            absolute_base_file = None
 
-        if not absolute_base_file or \
-                os.path.splitext(absolute_base_file)[1].lower() != '.shp':
-            raise Exception(
-                _(f"You are attempting to {action_type} a vector layer with an unknown format."))
-        else:
-            try:
-                gtype = layer.gtype if not gtype else gtype
-                inDataSource = ogr.Open(absolute_base_file)
-                lyr = inDataSource.GetLayer(str(layer.name))
-                if not lyr:
-                    raise Exception(
-                        _(f"Please ensure the name is consistent with the file you are trying to {action_type}."))
-                schema_is_compliant = False
-                _ff = json.loads(lyr.GetFeature(0).ExportToJson())
-                if gtype:
-                    logger.warning(
-                        _("Local GeoNode layer has no geometry type."))
-                    if _ff["geometry"]["type"] in gtype or gtype in _ff["geometry"]["type"]:
-                        schema_is_compliant = True
-                elif "geometry" in _ff and _ff["geometry"]["type"]:
+    if not absolute_base_file or \
+            os.path.splitext(absolute_base_file)[1].lower() != '.shp':
+        raise Exception(
+            _(f"You are attempting to {action_type} a vector layer with an unknown format."))
+    else:
+        try:
+            gtype = layer.gtype if not gtype else gtype
+            inDataSource = ogr.Open(absolute_base_file)
+            lyr = inDataSource.GetLayer(str(layer.name))
+            if not lyr:
+                raise Exception(
+                    _(f"Please ensure the name is consistent with the file you are trying to {action_type}."))
+            schema_is_compliant = False
+            _ff = json.loads(lyr.GetFeature(0).ExportToJson())
+            if gtype:
+                logger.info(
+                    _("Local GeoNode layer has geometry type."))
+                if _ff["geometry"]["type"] in gtype or gtype in _ff["geometry"]["type"]:
                     schema_is_compliant = True
+            elif "geometry" in _ff and _ff["geometry"]["type"]:
+                schema_is_compliant = True
 
-                if not schema_is_compliant:
-                    raise Exception(
-                        _(f"Please ensure there is at least one geometry type \
-                            that is consistent with the file you are trying to {action_type}."))
+            if not schema_is_compliant:
+                raise Exception(
+                    _(f"Please ensure that the geometry type \
+                        is consistent with the file you are trying to {action_type}."))
 
-                new_schema_fields = [field.name for field in lyr.schema]
-                gs_layer = gs_catalog.get_layer(layer.name)
+            new_schema_fields = [field.name for field in lyr.schema]
+            gs_layer = gs_catalog.get_layer(layer.name)
 
-                if not gs_layer:
-                    raise Exception(
-                        _("The selected Layer does not exists in the catalog."))
+            if not gs_layer:
+                raise Exception(
+                    _("The selected Layer does not exists in the catalog."))
 
-                gs_layer = gs_layer.resource.attributes
-                schema_is_compliant = all([x.replace("-", '_') in gs_layer for x in new_schema_fields])
+            gs_layer = gs_layer.resource.attributes
+            schema_is_compliant = all([x.replace("-", '_') in gs_layer for x in new_schema_fields])
 
-                if not schema_is_compliant:
-                    raise Exception(
-                        _("Please ensure that the layer structure is consistent "
-                          f"with the file you are trying to {action_type}."))
-                return True
-            except Exception as e:
+            if not schema_is_compliant:
                 raise Exception(
-                    _(f"Some error occurred while trying to access the uploaded schema: {str(e)}"))
+                    _("Please ensure that the layer structure is consistent "
+                        f"with the file you are trying to {action_type}."))
+            return True
+        except Exception as e:
+            raise Exception(
+                _(f"Some error occurred while trying to access the uploaded schema: {str(e)}"))
 
 
 def is_xml_upload_only(request):