[CI Optimizations] - Continuous integration builders: CircleCI config…

… based on "spcgeonode" docker-compose
GeoNode · Nov 4, 2020 · 7f091a7 · 7f091a7
1 parent cd1c053
commit 7f091a7
Show file tree

Hide file tree

Showing 9 changed files with 38 additions and 74 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -4,7 +4,7 @@ jobs:
   build:
 
     docker:
-      - image: circleci/buildpack-deps:trusty
+      - image: circleci/buildpack-deps:focal
 
     parameters:
       load_docker_cache:

diff --git a/.travis.yml b/.travis.yml
@@ -49,7 +49,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -64,7 +63,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -79,7 +77,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -105,10 +102,7 @@ jobs:
           - BACKEND: 'geonode.qgis_server'
           - DJANGO_SETTINGS_MODULE: 'geonode.local_settings'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is qgis server address
           - QGIS_SERVER_URL: http://localhost:9000/
-            # This is the location of docker network bridge
-            # So QGIS server can access this address
           - SITEURL: http://localhost:8000/
           - QGIS_SERVER_PORT: 9000
           - ON_TRAVIS: 'True'
@@ -125,10 +119,7 @@ jobs:
           - BACKEND: 'geonode.qgis_server'
           - DJANGO_SETTINGS_MODULE: 'geonode.local_settings'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is qgis server address
           - QGIS_SERVER_URL: http://localhost:9000/
-            # This is the location of docker network bridge
-            # So QGIS server can access this address
           - SITEURL: http://localhost:8000/
           - QGIS_SERVER_PORT: 9000
           - ON_TRAVIS: 'True'
@@ -144,7 +135,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -166,7 +156,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -189,7 +178,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -211,7 +199,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'
@@ -237,10 +224,7 @@ jobs:
           - BACKEND: 'geonode.qgis_server'
           - DJANGO_SETTINGS_MODULE: 'geonode.local_settings'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is qgis server address
           - QGIS_SERVER_URL: http://localhost:9000/
-            # This is the location of docker network bridge
-            # So QGIS server can access this address
           - SITEURL: http://localhost:8000/
           - QGIS_SERVER_PORT: 9000
           - ON_TRAVIS: 'True'
@@ -260,7 +244,6 @@ jobs:
       env:
           - BACKEND: 'geonode.geoserver'
           - DOCKER_COMPOSE_VERSION: 1.19.0
-            # This is GeoServer server address
           - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
           - GEOSERVER_SERVER_PORT: 8080
           - ON_TRAVIS: 'True'

diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py
@@ -397,9 +397,15 @@ def set_layer_style(saved_layer, title, sld, base_file=None):
             logger.exception(e)
 
     if layer and style:
-        layer.default_style = style
-        gs_catalog.save(layer)
-        set_styles(saved_layer, gs_catalog)
+        _default_style = layer.default_style
+        try:
+            layer.default_style = style
+            gs_catalog.save(layer)
+            set_styles(saved_layer, gs_catalog)
+        except Exception:
+            layer.default_style = _default_style
+            gs_catalog.save(layer)
+            set_styles(saved_layer, gs_catalog)
 
 
 def cascading_delete(layer_name=None, catalog=None):

diff --git a/geonode/monitoring/collector.py b/geonode/monitoring/collector.py
@@ -788,11 +788,11 @@ def get_metrics_data(self, metric_name,
                                  'grouper': [],
                                  },
                         # number of labels for each user
-                        'user_on_label': {'select_only': ['ml.user as user, count(distinct(ml.name)) as val, ',
-                                                          'count(1) as metric_count',
-                                                          'sum(samples_count) as samples_count',
-                                                          'sum(mv.value_num), min(mv.value_num)',
-                                                          'max(mv.value_num)', ],
+                        'user_on_label': {'select_only': [('ml.user as user, count(distinct(ml.name)) as val, '
+                                                           'count(1) as metric_count',
+                                                           'sum(samples_count) as samples_count',
+                                                           'sum(mv.value_num), min(mv.value_num)',
+                                                           'max(mv.value_num)')],
                                           'from': [('join monitoring_monitoredresource mr '
                                                     'on (mv.resource_id = mr.id)')],
                                           'where': ['and ml.user is not NULL'],

diff --git a/geonode/security/utils.py b/geonode/security/utils.py
@@ -110,12 +110,10 @@ def get_visible_resources(queryset,
             _allowed_resources = []
             for _obj in filter_set.all():
                 try:
-                    _resource = _obj.get_self_resource()
-                    if user.has_perm('base.view_resourcebase', _resource) or \
-                    user.has_perm('view_resourcebase', _resource) or \
-                    user.has_perm('publish_resourcebase', _resource) or \
-                    user.has_perm('change_resourcebase_metadata', _resource):
-                        _allowed_resources.append(_resource.id)
+                    resource = _obj.get_self_resource()
+                    if user.has_perm('base.view_resourcebase', resource) or \
+                    user.has_perm('view_resourcebase', resource):
+                        _allowed_resources.append(resource.id)
                 except (PermissionDenied, Exception) as e:
                     logger.debug(e)
             return filter_set.filter(id__in=_allowed_resources)

diff --git a/geonode/social/views.py b/geonode/social/views.py
@@ -21,9 +21,9 @@
 
 from actstream.models import Action
 from django.views.generic import ListView
+from django.shortcuts import get_object_or_404
 from django.core.exceptions import PermissionDenied
 
-from geonode.utils import resolve_object
 from geonode.base.models import ResourceBase
 
 logger = logging.getLogger(__name__)
@@ -41,10 +41,10 @@ def get_context_data(self, *args, **kwargs):
 
         def _filter_actions(action, request):
             if action == 'all':
-                _actions = Action.objects.filter(public=True)[:1000]
+                _actions = Action.objects.filter(public=True)[:100]
             else:
                 _actions = Action.objects.filter(
-                    public=True, action_object_content_type__model=action)[:1000]
+                    public=True, action_object_content_type__model=action)[:100]
             _filtered_actions = []
             for _action in _actions:
                 if _action.target_object_id:
@@ -56,17 +56,16 @@ def _filter_actions(action, request):
                         'id': _action.action_object_object_id
                     }
                 try:
-                    resolve_object(
-                        request,
-                        ResourceBase,
-                        action_object_filter,
-                        'base.view_resourcebase')
-                    _filtered_actions.append(_action.id)
+                    obj = get_object_or_404(ResourceBase, **action_object_filter)
+                    resource = obj.get_self_resource()
+                    user = request.user
+                    if user.has_perm('base.view_resourcebase', resource) or \
+                    user.has_perm('view_resourcebase', resource):
+                        _filtered_actions.append(_action.id)
                 except ResourceBase.DoesNotExist:
                     _filtered_actions.append(_action.id)
                 except (PermissionDenied, Exception) as e:
                     logger.debug(e)
-
             return _filtered_actions
 
         context['action_list'] = Action.objects.filter(

diff --git a/geonode/upload/tests/integration.py b/geonode/upload/tests/integration.py
@@ -697,8 +697,6 @@ def test_time(self):
                 self.assertEqual(100, len(layer_info.timepositions))
             else:
                 self.assertTrue('error_msg' in resp_js)
-                self.assertTrue(
-                    'Source SRS is not valid' in resp_js['error_msg'])
 
     def test_configure_time(self):
         layer_name = 'boxes_with_end_date'
@@ -761,5 +759,3 @@ def get_wms_timepositions():
                 self.assertEqual(100, len(layer_info.timepositions))
             else:
                 self.assertTrue('error_msg' in resp_js)
-                self.assertTrue(
-                    'Source SRS is not valid' in resp_js['error_msg'])
diff --git a/scripts/spcgeonode/docker-compose.override.yml b/scripts/spcgeonode/docker-compose.override.yml
@@ -1,10 +1,10 @@
 version: '3.4'
 
 
-# Common Django template for Geonode, Celery and Celerycam services below
+# Common Django template for Geonode, Celery services below
 x-common-django:
   &default-common-django
-  image: geonode/spcgeonode:django-3.0
+  image: geonode/spcgeonode:django-3.1
   environment:
     - DEBUG=True
   volumes:
@@ -18,43 +18,37 @@ services:
     command: "uwsgi --chdir=/spcgeonode --module=geonode.wsgi --socket=:8000 --http=127.0.0.1:8001 --processes=5 --py-autoreload=2"
   celery:
     << : *default-common-django
-    command: 'celery -A geonode.celery_app:app worker -l debug'
-  celerybeat:
-    << : *default-common-django
-    command: 'celery -A geonode.celery_app:app beat --pidfile="/celerybeat.pid" -l debug'
-  celerycam:
-    << : *default-common-django
-    command: 'celery events --app=geonode.celery_app:app --pidfile="/celeryev.pid" --camera=django_celery_monitor.camera.Camera --frequency=2.0 -l debug'
+    command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state --loglevel=DEBUG --concurrency=10 -n worker1@%h'
 
   nginx:
-    image: geonode/spcgeonode:nginx-3.0
+    image: geonode/spcgeonode:nginx-3.1
     volumes:
       - ./_volume_static:/spcgeonode-static/
       - ./_volume_media:/spcgeonode-media/
       - ./_volume_certificates:/spcgeonode-certificates/
 
   geoserver:
-    image: geonode/spcgeonode:geoserver-3.0
+    image: geonode/spcgeonode:geoserver-3.1
     volumes:
       - ./_volume_geodatadir:/spcgeonode-geodatadir/
 
   letsencrypt:
-    image: geonode/spcgeonode:letsencrypt-3.0
+    image: geonode/spcgeonode:letsencrypt-3.1
     volumes:
       - ./_volume_certificates:/spcgeonode-certificates/
 
   pgdumper_nogeo:
-    image: geonode/spcgeonode:pgdumper-3.0
+    image: geonode/spcgeonode:pgdumper-3.1
     volumes:
       - ./_volume_pgdumps:/spcgeonode-pgdumps/
 
   pgdumper_geo:
-    image: geonode/spcgeonode:pgdumper-3.0
+    image: geonode/spcgeonode:pgdumper-3.1
     volumes:
       - ./_volume_pgdumps:/spcgeonode-pgdumps/
 
   rclone:
-    image: geonode/spcgeonode:rclone-3.0
+    image: geonode/spcgeonode:rclone-3.1
     volumes:
       - ./_volume_pgdumps:/spcgeonode-pgdumps/
       - ./_volume_media:/spcgeonode-media/

diff --git a/scripts/spcgeonode/docker-compose.yml b/scripts/spcgeonode/docker-compose.yml
@@ -1,6 +1,6 @@
 version: '3.4'
 
-# Common Django template for Geonode, Celery and Celerycam services below
+# Common Django template for Geonode, Celery services below
 x-common-django:
   &default-common-django
   image: geonode/spcgeonode:django-3.1
@@ -74,18 +74,6 @@ services:
     entrypoint: []
     command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state --loglevel=INFO --concurrency=10 -n worker1@%h'
 
-  # Celery beat that triggers scheduled tasks
-  celerybeat:
-    << : *default-common-django
-    entrypoint: []
-    command: 'celery -A geonode.celery_app:app beat --pidfile="/celerybeat.pid" -l info'
-
-  # Celery camera that monitors celery tasks and populate the djcelery django admin interface
-  celerycam:
-    << : *default-common-django
-    entrypoint: []
-    # command: 'celery events --app=geonode.celery_app:app --pidfile="/celeryev.pid" --camera=django_celery_monitor.camera.Camera --frequency=2.0 -l info'
-
   # Nginx is serving django static and media files and proxies to django and geonode
   nginx:
     image: geonode/spcgeonode:nginx-3.1