diff --git a/Vagrantfile b/Vagrantfile index 83b194b..f36330c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -17,7 +17,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision "shell", path: "scripts/before_script.sh", args: "/vagrant" config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", "2048"] + vb.customize ["modifyvm", :id, "--memory", "3072"] vb.customize ["modifyvm", :id, "--cpus", "2"] vb.customize ["modifyvm", :id, "--ioapic", "on"] end diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index 499380b..a487440 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -1,18 +1,22 @@ import json import logging -from tastypie.fields import DictField, ListField, CharField, ToManyField, ForeignKey + +import celery +from django.conf.urls import url from django.contrib.auth import get_user_model -from tastypie.constants import ALL, ALL_WITH_RELATIONS -from tastypie.resources import ModelResource -from .models import UploadedData, UploadLayer, UploadFile +from tastypie import http from tastypie.authentication import SessionAuthentication from tastypie.authorization import Authorization -from tastypie.utils import trailing_slash -from tastypie import http -from django.conf.urls import url from tastypie.bundle import Bundle -from .tasks import import_object +from tastypie.constants import ALL, ALL_WITH_RELATIONS from tastypie.exceptions import ImmediateHttpResponse +from tastypie.fields import DictField, ListField, CharField, ToManyField, ForeignKey +from tastypie.resources import ModelResource +from tastypie.utils import trailing_slash + +from .models import UploadedData, UploadLayer, UploadFile +from .tasks import import_object + logger = logging.getLogger(__name__) @@ -82,10 +86,17 @@ def import_layer(self, request, pk=None, **kwargs): raise ImmediateHttpResponse(response=http.HttpBadRequest('Configuration options missing.')) uploaded_file = obj.upload_file + # If this layer has been imported before, clear the status of the previous import task. + obj.import_status = None + obj.task_id = None + obj.save() + import_result = import_object.delay(uploaded_file.id, configuration_options=configuration_options) # query the db again for this object since it may have been updated during the import obj = self.obj_get(bundle, pk=pk) + if import_result.state in celery.states.READY_STATES: + obj.import_status = import_result.status obj.task_id = import_result.id obj.save() diff --git a/osgeo_importer/handlers/__init__.py b/osgeo_importer/handlers/__init__.py index e443ad1..9c665c1 100644 --- a/osgeo_importer/handlers/__init__.py +++ b/osgeo_importer/handlers/__init__.py @@ -4,16 +4,7 @@ from osgeo_importer.inspectors import OGRFieldConverter, BigDateOGRFieldConverter -DEFAULT_IMPORT_HANDLERS = ['osgeo_importer.handlers.FieldConverterHandler', - 'osgeo_importer.handlers.geoserver.GeoserverPublishHandler', - 'osgeo_importer.handlers.geoserver.GeoserverPublishCoverageHandler', - 'osgeo_importer.handlers.geoserver.GeoServerTimeHandler', - 'osgeo_importer.handlers.geoserver.GeoWebCacheHandler', - 'osgeo_importer.handlers.geoserver.GeoServerBoundsHandler', - 'osgeo_importer.handlers.geoserver.GenericSLDHandler', - 'osgeo_importer.handlers.geonode.GeoNodePublishHandler', - 'osgeo_importer.handlers.geoserver.GeoServerStyleHandler', - 'osgeo_importer.handlers.geonode.GeoNodeMetadataHandler'] +DEFAULT_IMPORT_HANDLERS = [] IMPORT_HANDLERS = getattr(settings, 'IMPORT_HANDLERS', DEFAULT_IMPORT_HANDLERS) diff --git a/osgeo_importer/migrations/0007_auto_20161025_2130.py b/osgeo_importer/migrations/0007_auto_20161025_2130.py new file mode 100644 index 0000000..86bb559 --- /dev/null +++ b/osgeo_importer/migrations/0007_auto_20161025_2130.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import osgeo_importer.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osgeo_importer', '0006_auto_20161007_1723'), + ] + + operations = [ + migrations.AlterField( + model_name='uploadfile', + name='file', + field=models.FileField(max_length=1000, upload_to=b'uploads', validators=[osgeo_importer.models.validate_file_extension, osgeo_importer.models.validate_inspector_can_read]), + ), + ] diff --git a/osgeo_importer/migrations/0008_uploadlayer_import_status.py b/osgeo_importer/migrations/0008_uploadlayer_import_status.py new file mode 100644 index 0000000..26b1a93 --- /dev/null +++ b/osgeo_importer/migrations/0008_uploadlayer_import_status.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osgeo_importer', '0007_auto_20161025_2130'), + ] + + operations = [ + migrations.AddField( + model_name='uploadlayer', + name='import_status', + field=models.CharField(max_length=15, null=True, blank=True), + ), + ] diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index e287da6..b5cefa1 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -171,7 +171,14 @@ def __unicode__(self): class UploadFile(models.Model): upload = models.ForeignKey(UploadedData, null=True, blank=True) - file = models.FileField(upload_to="uploads", validators=[validate_file_extension, validate_inspector_can_read]) + file = models.FileField( + upload_to="uploads", + max_length=1000, + validators=[ + validate_file_extension, + validate_inspector_can_read + ] + ) file_type = models.CharField(max_length=50, null=True, blank=True) slug = models.SlugField(max_length=250, blank=True) @@ -203,6 +210,7 @@ class UploadLayer(models.Model): object_id = models.PositiveIntegerField(blank=True, null=True) layer = GenericForeignKey('content_type', 'object_id') configuration_options = JSONField(null=True) + import_status = models.CharField(max_length=15, blank=True, null=True) task_id = models.CharField(max_length=36, blank=True, null=True) feature_count = models.IntegerField(null=True, blank=True) layer_name = models.CharField(max_length=64, null=True) @@ -248,17 +256,21 @@ def description(self): @property def status(self): """ - Returns the status of a single map page. + Returns the status of the import of this UploadedLayer. """ - if self.task_id: - try: - state = TaskState.objects.get(task_id=self.task_id).state - return state - except ObjectDoesNotExist: - asyncres = AsyncResult(self.task_id) - logger.debug("Import task status: {}".format(asyncres.status)) - return AsyncResult(self.task_id).status - return 'UNKNOWN' + if self.import_status is not None: + s = self.import_status + else: + if self.task_id is not None: + try: + s = TaskState.objects.get(task_id=self.task_id).state + except ObjectDoesNotExist: + asyncres = AsyncResult(self.task_id) + logger.debug("Import task status: {}".format(asyncres.status)) + s = AsyncResult(self.task_id).status + else: + s = 'UNKNOWN' + return s class Meta: ordering = ('index',) diff --git a/osgeo_importer_prj/settings.py b/osgeo_importer_prj/settings.py index 4ffbb7c..156e79a 100644 --- a/osgeo_importer_prj/settings.py +++ b/osgeo_importer_prj/settings.py @@ -24,9 +24,23 @@ # # General Django development settings # - SITENAME = 'osgeo_importer_prj' +IMPORT_HANDLERS = [ + # If GeoServer handlers are enabled, you must have an instance of geoserver running. + # Warning: the order of the handlers here matters. + 'osgeo_importer.handlers.FieldConverterHandler', + 'osgeo_importer.handlers.geoserver.GeoserverPublishHandler', + 'osgeo_importer.handlers.geoserver.GeoserverPublishCoverageHandler', + 'osgeo_importer.handlers.geoserver.GeoServerTimeHandler', + 'osgeo_importer.handlers.geoserver.GeoWebCacheHandler', + 'osgeo_importer.handlers.geoserver.GeoServerBoundsHandler', + 'osgeo_importer.handlers.geoserver.GenericSLDHandler', + 'osgeo_importer.handlers.geonode.GeoNodePublishHandler', + 'osgeo_importer.handlers.geoserver.GeoServerStyleHandler', + 'osgeo_importer.handlers.geonode.GeoNodeMetadataHandler' +] + # Defines the directory that contains the settings file as the LOCAL_ROOT # It is used for relative settings elsewhere. LOCAL_ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -82,7 +96,7 @@ OSGEO_IMPORTER_GEONODE_ENABLED = True OSGEO_IMPORTER_VALID_EXTENSIONS = [ 'shp', 'shx', 'prj', 'dbf', 'kml', 'geojson', 'json', 'tif', 'tiff', - 'gpkg', 'csv','zip','xml','sld' + 'gpkg', 'csv', 'zip', 'xml', 'sld' ] LOGGING['loggers']['osgeo_importer'] = {"handlers": ["console"], "level": "DEBUG"} DATABASE_ROUTERS = ['osgeo_importer_prj.dbrouters.DefaultOnlyMigrations']