Change GeoNodePublishHandler to create / update Layer instances without using geoserver-specific gs_slurp()

osgeo_importer/admin.py

@@ -14,6 +14,7 @@ class UploadedDataAdmin(admin.ModelAdmin):
     list_display = ('name', 'user', 'state', 'size', 'complete')
     list_filter = ('user', 'state', 'complete')
 
+
 admin.site.register(UploadLayer, UploadedLayerAdmin)
 admin.site.register(UploadedData, UploadedDataAdmin)
 admin.site.register(UploadFile, UploadAdmin)

osgeo_importer/handlers/geonode/__init__.py

@@ -1,78 +1,12 @@
+import os
+
+from geonode.layers.metadata import set_metadata
 from geonode.layers.models import Layer
-from osgeo_importer.models import UploadLayer
+from geonode.layers.utils import resolve_regions
 from osgeo_importer.handlers import ImportHandlerMixin
 from osgeo_importer.handlers import ensure_can_run
 from osgeo_importer.importers import UPLOAD_DIR
-from geonode.geoserver.helpers import gs_slurp
-from geonode.layers.metadata import set_metadata
-from geonode.layers.utils import resolve_regions
-from django.contrib.auth import get_user_model
-from django.conf import settings
-from django import db
-import os
-import logging
-
-User = get_user_model()
-logger = logging.getLogger(__name__)
-
-
-class GeoNodePublishHandler(ImportHandlerMixin):
-    """
-    Creates a GeoNode Layer from a layer in Geoserver.
-    """
-
-    workspace = 'geonode'
-
-    @property
-    def store_name(self):
-        geoserver_publishers = self.importer.filter_handler_results('GeoserverPublishHandler')
-
-        for result in geoserver_publishers:
-            for key, feature_type in result.items():
-                if feature_type and hasattr(feature_type, 'store'):
-                    return feature_type.store.name
-
-        return db.connections[settings.OSGEO_DATASTORE].settings_dict['NAME']
-
-    def can_run(self, layer, layer_config, *args, **kwargs):
-        """
-        Skips this layer if the user is appending data to another dataset.
-        """
-        return 'appendTo' not in layer_config
-
-    @ensure_can_run
-    def handle(self, layer, layer_config, *args, **kwargs):
-        """
-        Adds a layer in GeoNode, after it has been added to Geoserver.
-
-        Handler specific params:
-        "layer_owner": Sets the owner of the layer.
-        """
-        owner = layer_config.get('layer_owner')
-        if isinstance(owner, str) or isinstance(owner, unicode):
-            owner = User.objects.filter(username=owner).first()
-
-        if layer_config.get('raster'):
-            store_name = os.path.splitext(os.path.basename(layer))[0]
-            filter = None
-        else:
-            store_name = self.store_name
-            filter = layer
-
-        results = gs_slurp(workspace=self.workspace,
-                           store=store_name,
-                           filter=filter,
-                           owner=owner,
-                           permissions=layer_config.get('permissions'))
-
-        if self.importer.upload_file and results['layers'][0]['status'] == 'created':
-            matched_layer = Layer.objects.get(name=results['layers'][0]['name'])
-            upload_layer = UploadLayer.objects.get(upload_file=self.importer.upload_file.pk,
-                                                   index=layer_config.get('index'))
-            upload_layer.layer = matched_layer
-            upload_layer.save()
-
-        return results
+from publish_handler import GeoNodePublishHandler  # NOQA - Moved this code but want it still available here.
 
 
 class GeoNodeMetadataHandler(ImportHandlerMixin):

osgeo_importer/handlers/geonode/backward_compatibility.py

@@ -0,0 +1,100 @@
+import datetime
+import logging
+
+from geonode.layers.models import Attribute
+
+
+logger = logging.getLogger(__name__)
+
+# The function set_attributes() is used in GeoNodePublishHandler.handle() .
+# In case the user is using osgeo_importer with a version of GeoNode older than
+# 2016-11-08, commit 005bd559ed97ace87b33d7abd3e84d3e307d4547, duplicate the
+# GeoNode-independent set_attributes() here.  If this duplicated code is still
+# here after 2018-11-01, it should be removed and imports of set_attributes()
+# should changed to import from geonode.utils instead of here.
+try:
+    from geonode.utils import set_attributes
+except ImportError:
+    set_attributes = None
+
+
+def set_attributes_bw_compat(layer, attribute_map, overwrite=False, attribute_stats=None):
+    """ *layer*: a geonode.layers.models.Layer instance
+        *attribute_map*: a list of 2-lists specifying attribute names and types,
+            example: [ ['id', 'Integer'], ... ]
+        *overwrite*: replace existing attributes with new values if name/type matches.
+        *attribute_stats*: dictionary of return values from get_attribute_statistics(),
+            of the form to get values by referencing attribute_stats[<layer_name>][<field_name>].
+    """
+    # we need 3 more items; description, attribute_label, and display_order
+    attribute_map_dict = {
+        'field': 0,
+        'ftype': 1,
+        'description': 2,
+        'label': 3,
+        'display_order': 4,
+    }
+    for attribute in attribute_map:
+        attribute.extend((None, None, 0))
+
+    attributes = layer.attribute_set.all()
+    # Delete existing attributes if they no longer exist in an updated layer
+    for la in attributes:
+        lafound = False
+        for attribute in attribute_map:
+            field, ftype, description, label, display_order = attribute
+            if field == la.attribute:
+                lafound = True
+                # store description and attribute_label in attribute_map
+                attribute[attribute_map_dict['description']] = la.description
+                attribute[attribute_map_dict['label']] = la.attribute_label
+                attribute[attribute_map_dict['display_order']] = la.display_order
+        if overwrite or not lafound:
+            logger.debug(
+                "Going to delete [%s] for [%s]",
+                la.attribute,
+                layer.name.encode('utf-8'))
+            la.delete()
+
+    # Add new layer attributes if they don't already exist
+    if attribute_map is not None:
+        iter = len(Attribute.objects.filter(layer=layer)) + 1
+        for attribute in attribute_map:
+            field, ftype, description, label, display_order = attribute
+            if field is not None:
+                la, created = Attribute.objects.get_or_create(
+                    layer=layer, attribute=field, attribute_type=ftype,
+                    description=description, attribute_label=label,
+                    display_order=display_order)
+                if created:
+                    if (not attribute_stats or layer.name not in attribute_stats or
+                            field not in attribute_stats[layer.name]):
+                        result = None
+                    else:
+                        result = attribute_stats[layer.name][field]
+
+                    if result is not None:
+                        logger.debug("Generating layer attribute statistics")
+                        la.count = result['Count']
+                        la.min = result['Min']
+                        la.max = result['Max']
+                        la.average = result['Average']
+                        la.median = result['Median']
+                        la.stddev = result['StandardDeviation']
+                        la.sum = result['Sum']
+                        la.unique_values = result['unique_values']
+                        la.last_stats_updated = datetime.datetime.now()
+                    la.visible = ftype.find("gml:") != 0
+                    la.display_order = iter
+                    la.save()
+                    iter += 1
+                    logger.debug(
+                        "Created [%s] attribute for [%s]",
+                        field,
+                        layer.name.encode('utf-8'))
+    else:
+        logger.debug("No attributes found")
+
+
+if set_attributes is None:
+    set_attributes = set_attributes_bw_compat

osgeo_importer/handlers/geonode/publish_handler.py

@@ -0,0 +1,130 @@
+import os
+import uuid
+import logging
+
+from django import db
+from django.conf import settings
+from osgeo_importer.handlers import ImportHandlerMixin
+from osgeo_importer.handlers import ensure_can_run
+from osgeo_importer.models import UploadLayer
+from geonode.layers.models import Layer
+from backward_compatibility import set_attributes
+from django.contrib.auth import get_user_model
+
+User = get_user_model()
+logger = logging.getLogger(__name__)
+
+
+class GeoNodePublishHandler(ImportHandlerMixin):
+    """
+    Creates a GeoNode Layer from a layer in Geoserver.
+    """
+
+    workspace = 'geonode'
+
+    @property
+    def store_name(self):
+        geoserver_publishers = self.importer.filter_handler_results('GeoserverPublishHandler')
+
+        for result in geoserver_publishers:
+            for key, feature_type in result.items():
+                if feature_type and hasattr(feature_type, 'store'):
+                    return feature_type.store.name
+
+        return db.connections[settings.OSGEO_DATASTORE].settings_dict['NAME']
+
+    def can_run(self, layer, layer_config, *args, **kwargs):
+        """
+        Skips this layer if the user is appending data to another dataset.
+        """
+        return 'appendTo' not in layer_config
+
+    @ensure_can_run
+    def handle(self, layer, layer_config, *args, **kwargs):
+        """
+        Adds a layer in GeoNode, after it has been added to Geoserver.
+
+        Handler specific params:
+        "layer_owner": Sets the owner of the layer.
+        """
+        try:
+            owner = User.objects.get(username=layer_config['layer_owner'])
+        except KeyError:
+            owner = User.objects.get(username='AnonymousUser')
+
+        # Populate arguments to create a new Layer
+        if layer_config.get('raster'):
+            layer_name = os.path.splitext(os.path.basename(layer))[0]
+            store_name = layer_name
+            store_type = 'coverageStore'
+            fields = None
+        else:
+            layer_name = layer
+            store_name = self.store_name
+            store_type = 'dataStore'
+            fields = layer_config['fields']
+
+        workspace_name = self.workspace
+        layer_uuid = str(uuid.uuid4())
+        typename = '{}:{}'.format(workspace_name.encode('utf-8'), layer_name.encode('utf-8'))
+
+        new_layer_kwargs = {
+            'name': layer_name,
+            'workspace': self.workspace,
+            'store': store_name,
+            'storeType': store_type,
+            'typename': typename,
+            'title': layer_name,
+            "abstract": 'No abstract provided',
+            'owner': owner,
+            'uuid': layer_uuid,
+        }
+
+        new_layer, created = Layer.objects.get_or_create(**new_layer_kwargs)
+
+        # *** It is unclear where the date/time attributes are being created as part of the
+        # ***    above get_or_create().  It's probably a geoserver-specific save signal handler,
+        # ***    but until it gets tracked down, this will keep set_attributes() from
+        # ***    removing them since tests expect to find them.  set_attributes()
+        # ***    removes them if they aren't in the fields dict because it thinks
+        # ***    the layer is being updated and the new value doesn't include those attributes.
+        keep_attributes = ['date', 'date_as_date', 'enddate_as_date', 'time_as_date']
+        for a in new_layer.attributes:
+            if a.attribute in keep_attributes and a.attribute not in layer_config['fields']:
+                layer_config['fields'].append({'name': a.attribute, 'type': a.attribute_type})
+
+        # Add fields to new_layer.attribute_set
+        if fields:
+            attribute_map = [[f['name'], f['type']] for f in fields]
+            set_attributes(new_layer, attribute_map)
+
+        if self.importer.upload_file and created:
+            upload_layer = UploadLayer.objects.get(upload_file=self.importer.upload_file.pk,
+                                                   index=layer_config.get('index'))
+            upload_layer.layer = new_layer
+            upload_layer.save()
+
+        if 'permissions' in layer_config:
+            new_layer.set_permissions(layer_config['permissions'])
+        else:
+            new_layer.set_default_permissions()
+
+        results = {
+            'stats': {
+                'failed': 0,
+                'updated': 0,
+                'created': 0,
+                'deleted': 0,
+            },
+            'layers': [],
+            'deleted_layers': []
+        }
+
+        if created:
+            results['stats']['created'] += 1
+        else:
+            results['stats']['updated'] += 1
+
+        layer_info = {'name': layer, 'status': 'created' if created else 'updated'}
+        results['layers'].append(layer_info)
+        return results

osgeo_importer/tests/handlers/geonode/test_backward_compatibility.py

@@ -0,0 +1,59 @@
+import copy
+
+from django.contrib.auth import get_user_model
+from django.db.models import signals
+from django.test import TestCase
+from geonode.geoserver.signals import geoserver_post_save
+
+from geonode.layers.models import Layer
+from osgeo_importer.handlers.geonode.backward_compatibility import set_attributes_bw_compat as set_attributes
+
+
+class TestSetAttributes(TestCase):
+    """ This is copied & modified from geonode.tests.utils.
+        @see backward_compatibility for details.
+    """
+    def setUp(self):
+        # Load users to log in as
+        # call_command('loaddata', 'people_data', verbosity=0)
+        User = get_user_model()
+        User.objects.create_superuser(username='norman', password='norman', email='')
+
+    def test_set_attributes_creates_attributes(self):
+        """ Test utility function set_attributes() which creates Attribute instances attached
+            to a Layer instance.
+        """
+        # Creating a layer requires being logged in
+        self.client.login(username='norman', password='norman')
+
+        # Disconnect the geoserver-specific post_save signal attached to Layer creation.
+        # The geoserver signal handler assumes things about the store where the Layer is placed.
+        # this is a workaround.
+        disconnected_post_save = signals.post_save.disconnect(geoserver_post_save, sender=Layer)
+
+        # Create dummy layer to attach attributes to
+        l = Layer.objects.create(name='dummy_layer')
+
+        # Reconnect the signal if it was disconnected
+        if disconnected_post_save:
+            signals.post_save.connect(geoserver_post_save, sender=Layer)
+
+        attribute_map = [
+            ['id', 'Integer'],
+            ['date', 'IntegerList'],
+            ['enddate', 'Real'],
+            ['date_as_date', 'xsd:dateTime'],
+        ]
+
+        # attribute_map gets modified as a side-effect of the call to set_attributes()
+        expected_results = copy.deepcopy(attribute_map)
+
+        # set attributes for resource
+        set_attributes(l, attribute_map)
+
+        # 2 items in attribute_map should translate into 2 Attribute instances
+        self.assertEquals(l.attributes.count(), len(expected_results))
+
+        # The name and type should be set as provided by attribute map
+        for a in l.attributes:
+            self.assertIn([a.attribute, a.attribute_type], expected_results)