[Fixes GeoNode#8855] Plugged service type are ignored in the model validation (GeoNode#8863)

* [Fixes GeoNode#8855] Plugged service type are ignored in the model validation

* - Make sure we are going to use the service GetCapa operation to fetch the FT attributes

Author: Alessio Fabiani

geonode/base/migrations/0079_alter_resourcebase_alternate.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-02-25 16:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0078_alter_resourcebase_metadata'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='resourcebase',
+            name='alternate',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

geonode/base/models.py

@@ -797,7 +797,7 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase):
     contacts = models.ManyToManyField(
         settings.AUTH_USER_MODEL,
         through='ContactRole')
-    alternate = models.CharField(max_length=128, null=True, blank=True)
+    alternate = models.CharField(max_length=255, null=True, blank=True)
     date = models.DateTimeField(
         _('date'),
         default=now,

geonode/geoserver/helpers.py

@@ -979,7 +979,15 @@ def set_attributes_from_geoserver(layer, overwrite=False):
     then store in GeoNode database using Attribute model
     """
     attribute_map = []
-    server_url = ogc_server_settings.LOCATION if layer.subtype not in ['tileStore', 'remote'] else layer.remote_service.service_url
+    if getattr(layer, 'remote_service') and layer.remote_service:
+        server_url = layer.remote_service.service_url
+        if layer.remote_service.operations.get('GetCapabilities', None) and layer.remote_service.operations.get('GetCapabilities').get('methods'):
+            for _method in layer.remote_service.operations.get('GetCapabilities').get('methods'):
+                if _method.get('type', '').upper() == 'GET':
+                    server_url = _method.get('url', server_url)
+                    break
+    else:
+        server_url = ogc_server_settings.LOCATION
     if layer.subtype in ['tileStore', 'remote'] and layer.remote_service.ptype == "gxp_arcrestsource":
         dft_url = f"{server_url}{(layer.alternate or layer.typename)}?f=json"
         try:
@@ -994,13 +1002,16 @@ def set_attributes_from_geoserver(layer, overwrite=False):
             attribute_map = []
     elif layer.subtype in {"vector", "tileStore", "remote", "wmsStore"}:
         typename = layer.alternate if layer.alternate else layer.typename
-        dft_url = re.sub(r"\/wms\/?$",
-                         "/",
-                         server_url) + "ows?" + urlencode({"service": "wfs",
-                                                           "version": "1.0.0",
-                                                           "request": "DescribeFeatureType",
-                                                           "typename": typename,
-                                                           })
+        dft_url_path = re.sub(r"\/wms\/?$", "/", server_url)
+        dft_query = urlencode(
+            {
+                "service": "wfs",
+                "version": "1.0.0",
+                "request": "DescribeFeatureType",
+                "typename": typename
+            }
+        )
+        dft_url = urljoin(dft_url_path, f"ows?{dft_query}")
         try:
             # The code below will fail if http_client cannot be imported or WFS not supported
             req, body = http_client.get(dft_url, user=_user)

geonode/layers/enumerations.py

@@ -45,4 +45,5 @@
     'REST_IMG': 'gxp_arcrestsource',
     'HGL': 'gxp_hglsource',
     'GN_WMS': 'gxp_wmscsource',
+    'SOS': 'gxp_sos',
 }

geonode/layers/migrations/0043_auto_20220225_1605.py

@@ -0,0 +1,38 @@
+# Generated by Django 3.2.12 on 2022-02-25 16:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layers', '0042_alter_dataset_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataset',
+            name='name',
+            field=models.CharField(max_length=255, verbose_name='Name'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='ptype',
+            field=models.CharField(default='gxp_wmscsource', max_length=255, verbose_name='P-Type'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='store',
+            field=models.CharField(max_length=255, verbose_name='Store'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='typename',
+            field=models.CharField(blank=True, max_length=255, null=True, verbose_name='Typename'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='workspace',
+            field=models.CharField(max_length=255, verbose_name='Workspace'),
+        ),
+    ]

geonode/layers/models.py

@@ -128,10 +128,10 @@ class Dataset(ResourceBase):
 
     # internal fields
     objects = DatasetManager()
-    workspace = models.CharField(_('Workspace'), max_length=128)
-    store = models.CharField(_('Store'), max_length=128)
-    name = models.CharField(_('Name'), max_length=128)
-    typename = models.CharField(_('Typename'), max_length=128, null=True, blank=True)
+    workspace = models.CharField(_('Workspace'), max_length=255)
+    store = models.CharField(_('Store'), max_length=255)
+    name = models.CharField(_('Name'), max_length=255)
+    typename = models.CharField(_('Typename'), max_length=255, null=True, blank=True)
     ows_url = models.URLField(
         _('ows URL'),
         null=True,
@@ -153,7 +153,7 @@ class Dataset(ResourceBase):
         _('P-Type'),
         null=False,
         blank=False,
-        max_length=80,
+        max_length=255,
         default="gxp_wmscsource")
 
     default_style = models.ForeignKey(

geonode/resource/manager.py

@@ -69,7 +69,6 @@
     resourcebase_post_save)
 
 from ..base import enumerations
-from ..services.models import Service
 from ..base.models import ResourceBase
 from ..layers.metadata import parse_metadata
 from ..documents.models import Document, DocumentResourceLink
@@ -296,6 +295,8 @@ def delete(self, uuid: str, /, instance: ResourceBase = None) -> int:
                     logger.exception(e)
 
                 try:
+                    from ..services.models import Service
+
                     if _resource.remote_typename and Service.objects.filter(name=_resource.remote_typename).exists():
                         _service = Service.objects.filter(name=_resource.remote_typename).get()
                         if _service.harvester:

geonode/resource/utils.py

@@ -56,9 +56,6 @@
 from ..layers.utils import resolve_regions
 from ..layers.metadata import convert_keyword
 
-from ..services.models import Service
-from ..harvesting.models import HarvestableResource
-
 logger = logging.getLogger(__name__)
 
 ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)['default']
@@ -245,6 +242,9 @@ def update_resource(instance: ResourceBase, xml_file: str = None, regions: list
         logger.error(f"{e} - {to_update}")
 
     # Check for "remote services" availability
+    from ..services.models import Service
+    from ..harvesting.models import HarvestableResource
+
     if HarvestableResource.objects.filter(geonode_resource__uuid=instance.uuid).exists():
         _h = HarvestableResource.objects.filter(geonode_resource__uuid=instance.uuid).get().harvester
         if Service.objects.filter(harvester=_h).exists():

geonode/services/enumerations.py

@@ -17,9 +17,6 @@
 #
 #########################################################################
 
-from django.utils.translation import ugettext_lazy as _
-
-
 AUTO = "AUTO"
 OWS = "OWS"
 WMS = "WMS"
@@ -40,19 +37,6 @@
 LIVE = "X"
 OPENGEOPORTAL = "O"
 
-SERVICE_TYPES = (
-    # (AUTO, _('Auto-detect')),
-    (OWS, _('Paired WMS/WFS/WCS')),
-    (WMS, _('Web Map Service')),
-    # (CSW, _('Catalogue Service')),
-    (REST_MAP, _('ArcGIS REST MapServer')),
-    (REST_IMG, _('ArcGIS REST ImageServer')),
-    # (OGP, _('OpenGeoPortal')),
-    # (HGL, _('Harvard Geospatial Library')),
-    (GN_WMS, _('GeoNode (Web Map Service)')),
-    # (GN_CSW, _('GeoNode (Catalogue Service)')),
-)
-
 HARVESTER_TYPES = {
     'WMS': 'geonode.harvesting.harvesters.wms.OgcWmsHarvester',
     'GN_WMS': 'geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker',

geonode/services/forms.py

@@ -27,7 +27,7 @@
 from . import enumerations
 from .models import Service
 from .serviceprocessors import get_service_handler
-from geonode.services.serviceprocessors.handler import get_available_service_type
+from geonode.services.serviceprocessors import get_available_service_types
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +48,7 @@ class CreateServiceForm(forms.Form):
     )
     type = forms.ChoiceField(
         label=_("Service Type"),
-        choices=[(k, v["label"]) for k, v in get_available_service_type().items()],  # from dictionary to tuple
+        choices=[(k, v["label"]) for k, v in get_available_service_types().items()],  # from dictionary to tuple
         initial='AUTO',
     )
 

geonode/services/migrations/0054_alter_service_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-02-25 16:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('services', '0053_alter_service_type'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='service',
+            name='type',
+            field=models.CharField(choices=[('WMS', 'Web Map Service'), ('GN_WMS', 'GeoNode (Web Map Service)'), ('REST_MAP', 'ArcGIS REST MapServer'), ('REST_IMG', 'ArcGIS REST ImageServer')], max_length=10),
+        ),
+    ]

geonode/services/models.py

@@ -30,18 +30,21 @@
 from geonode.harvesting.models import Harvester
 from geonode.layers.enumerations import GXP_PTYPES
 from geonode.people.enumerations import ROLE_VALUES
+from geonode.services.serviceprocessors import get_available_service_types
 
 from . import enumerations
 
+service_type_as_tuple = [(k, v["label"]) for k, v in get_available_service_types().items()]
+
 logger = logging.getLogger("geonode.services")
 
 
 class Service(ResourceBase):
     """Service Class to represent remote Geo Web Services"""
 
     type = models.CharField(
-        max_length=100,
-        choices=enumerations.SERVICE_TYPES
+        max_length=10,
+        choices=service_type_as_tuple
     )
     method = models.CharField(
         max_length=1,
@@ -126,7 +129,7 @@ def ptype(self):
     @property
     def service_type(self):
         # Return the gxp ptype that should be used to display layers
-        return [x for x in enumerations.SERVICE_TYPES if x[0] == self.type][0][1]
+        return [x for x in service_type_as_tuple if x[0] == self.type][0][1]
 
     def get_absolute_url(self):
         return '/services/%i' % self.id

geonode/services/serviceprocessors/__init__.py

@@ -16,5 +16,48 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
+import logging
 
-from .handler import get_service_handler  # noqa
+from collections import OrderedDict
+from django.utils.translation import ugettext as _
+
+from geonode.services import enumerations
+from geonode.services.utils import parse_services_types
+
+logger = logging.getLogger(__name__)
+
+
+def get_available_service_types():
+    # LGTM: Fixes - Module uses member of cyclically imported module, which can lead to failure at import time.
+    from geonode.services.serviceprocessors.wms import GeoNodeServiceHandler, WmsServiceHandler
+    from geonode.services.serviceprocessors.arcgis import ArcImageServiceHandler, ArcMapServiceHandler
+
+    default = OrderedDict({
+        enumerations.WMS: {"OWS": True, "handler": WmsServiceHandler, "label": _('Web Map Service')},
+        enumerations.GN_WMS: {"OWS": True, "handler": GeoNodeServiceHandler, "label": _('GeoNode (Web Map Service)')},
+        # enumerations.WFS: {"OWS": True, "handler": ServiceHandlerBase, "label": _('Paired WMS/WFS/WCS'},
+        # enumerations.TMS: {"OWS": False, "handler": ServiceHandlerBase, "label": _('Paired WMS/WFS/WCS'},
+        enumerations.REST_MAP: {"OWS": False, "handler": ArcMapServiceHandler, "label": _('ArcGIS REST MapServer')},
+        enumerations.REST_IMG: {"OWS": False, "handler": ArcImageServiceHandler, "label": _('ArcGIS REST ImageServer')},
+        # enumerations.CSW: {"OWS": False, "handler": ServiceHandlerBase, "label": _('Catalogue Service')},
+        # enumerations.OGP: {"OWS": True, "handler": ServiceHandlerBase, "label": _('OpenGeoPortal')},  # TODO: verify this
+        # enumerations.HGL: {"OWS": False, "handler": ServiceHandlerBase, "label": _('Harvard Geospatial Library')},  # TODO: verify this
+    })
+
+    return OrderedDict({**default, **parse_services_types()})
+
+
+def get_service_handler(base_url, service_type=enumerations.AUTO, service_id=None):
+    """Return the appropriate remote service handler for the input URL.
+    If the service type is not explicitly passed in it will be guessed from
+    """
+    handlers = get_available_service_types()
+
+    handler = handlers.get(service_type, {}).get("handler")
+    try:
+        service = handler(base_url, service_id)
+    except Exception:
+        logger.exception(
+            msg=f"Could not parse service {base_url}")
+        raise
+    return service

geonode/services/serviceprocessors/arcgis.py

@@ -99,7 +99,7 @@ def probe(self):
     def create_cascaded_store(self, service):
         return None
 
-    def create_geonode_service(self, owner):
+    def create_geonode_service(self, owner, parent=None):
         """Create a new geonode.service.models.Service instance
 
         :arg owner: The user who will own the service instance

geonode/services/serviceprocessors/base.py

@@ -85,7 +85,7 @@ def probe(self):
     def get_harvester_configuration_options(self):
         return {}
 
-    def create_geonode_service(self, owner):
+    def create_geonode_service(self, owner, parent=None):
         """Create a new geonode.service.models.Service instance
         Saving the service instance in the database is not a concern of this
         method, it only deals with creating the instance.