Merge branch 'master' into spcgeonode-remove-extra-postgres-wait

geonode/api/resourcebase_api.py

@@ -325,6 +325,9 @@ def build_haystack_filters(self, parameters):
                 elif type in LAYER_SUBTYPES.keys():
                     subtypes.append(type)
 
+            if 'vector' in subtypes and 'vector_time' not in subtypes:
+                subtypes.append('vector_time')
+
             if len(subtypes) > 0:
                 types.append("layer")
                 sqs = SearchQuerySet().narrow("subtype:%s" %

geonode/geoserver/tasks.py

@@ -32,7 +32,7 @@ def geoserver_update_layers(self, *args, **kwargs):
     return gs_slurp(*args, **kwargs)
 
 
-@shared_task(bind=True)
+@shared_task(bind=True, queue='update')
 def thumbnail_task(self,
                    instance_id,
                    instance_type,

geonode/layers/search_indexes.py

@@ -99,7 +99,7 @@ def prepare_type(self, obj):
     def prepare_subtype(self, obj):
         if obj.storeType == "dataStore":
             if obj.has_time:
-                return "vectorTimeSeries"
+                return "vector_time"
             else:
                 return "vector"
         elif obj.storeType == "coverageStore":

geonode/static/geonode/js/upload/LayerInfo.js

@@ -303,7 +303,7 @@ define(function (require, exports) {
      *  @returns {string}
      */
     LayerInfo.prototype.markError = function (error, status) {
-        var error = (error != undefined ? error : 'Unespected error!');
+        var error = (error != undefined ? error : 'Unexpected error!');
         common.logError(error, this.element.find('#status'));
     };
 

geonode/upload/utils.py

@@ -742,10 +742,17 @@ def make_geogig_rest_payload(author_name='admin',
     if settings.OGC_SERVER['default']['PG_GEOGIG'] is True:
         datastore = settings.OGC_SERVER['default']['DATASTORE']
         pg_geogig_db = settings.DATABASES[datastore]
+        schema = 'public'
+        if 'OPTIONS' in pg_geogig_db and 'options' in pg_geogig_db['OPTIONS']:
+            detected_schemas = re.findall('((?<=search_path=).*)\s?',
+                                          pg_geogig_db['OPTIONS']['options'])
+            if len(detected_schemas) > 0:
+                schemas = detected_schemas[0]
+                schema = schemas.split(',')[0]
         payload["dbHost"] = pg_geogig_db['HOST']
         payload["dbPort"] = pg_geogig_db.get('PORT', '5432')
         payload["dbName"] = pg_geogig_db['NAME']
-        payload["dbSchema"] = pg_geogig_db.get('SCHEMA', 'public')
+        payload["dbSchema"] = schema
         payload["dbUser"] = pg_geogig_db['USER']
         payload["dbPassword"] = pg_geogig_db['PASSWORD']
     else:

scripts/spcgeonode/.env

@@ -26,6 +26,9 @@ ADMIN_USERNAME=super
 ADMIN_PASSWORD=duper
 ADMIN_EMAIL=[email protected]
 
+# PostgreSQL superuser password (to be set if PostgreSQL is exposed)
+POSTGRES_PASSWORD=
+
 # Django secret key (replace this by any complex and random string)
 SECRET_KEY=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ
 

scripts/spcgeonode/docker-compose.yml

@@ -25,7 +25,7 @@ x-common-django:
     # hardcoded
     - DEBUG=False
     - DJANGO_SETTINGS_MODULE=geonode.settings
-    - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres
+    - DATABASE_URL=postgres://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
     - BROKER_URL=amqp://rabbitmq:5672
     - STATIC_ROOT=/spcgeonode-static/
     - MEDIA_ROOT=/spcgeonode-media/
@@ -109,6 +109,7 @@ services:
       - HTTP_PORT=${HTTP_PORT}
       - ADMIN_USERNAME=${ADMIN_USERNAME}
       - ADMIN_PASSWORD=${ADMIN_PASSWORD}
+      - DATABASE_URL=postgres://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
     volumes:
       - geodatadir:/spcgeonode-geodatadir/
     restart: on-failure
@@ -129,6 +130,8 @@ services:
   pgdumper:
     image: olivierdalang/spcgeonode:pgdumper-latest
     build: ./pgdumper/
+    environment:
+      - DATABASE_URL=postgres://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
     volumes:
       - pgdumps:/spcgeonode-pgdumps/ 
     restart: on-failure
@@ -150,6 +153,8 @@ services:
   # PostGIS database.
   postgres:
     image: mdillon/postgis:9.6-alpine
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
     volumes:
       - database:/var/lib/postgresql/data/ 
     restart: on-failure

scripts/spcgeonode/geoserver/Dockerfile

@@ -1,5 +1,9 @@
 FROM openjdk:8-jre-alpine
 
+ARG version=2.14.2
+ARG branch=2.14.x
+ARG plugins=
+
 # Install dependencies
 RUN apk add --no-cache ca-certificates openssl curl postgresql-client fontconfig ttf-ubuntu-font-family
 RUN update-ca-certificates
@@ -12,20 +16,30 @@ WORKDIR /
 # TODO : this is a bit dirty..... can't we stat from vanilla Geoserver ?
 # TODO : merge into on step
 RUN echo "Download geoserver for geonode" && \
-    wget https://downloads.sourceforge.net/project/geoserver/GeoServer/2.14.0/geoserver-2.14.0-bin.zip && \
-    wget https://build.geo-solutions.it/geonode/geoserver/latest/geoserver-2.14.x.war --no-check-certificate && \
-    unzip geoserver-2.14.0-bin.zip && \
-    rm /geoserver-2.14.0-bin.zip && \
-    rm /geoserver-2.14.0/webapps/geoserver/* -rf && \
-    unzip -o geoserver-2.14.x.war -d /geoserver-2.14.0/webapps/geoserver/ && \
-    rm /geoserver-2.14.x.war
+    wget https://downloads.sourceforge.net/project/geoserver/GeoServer/$version/geoserver-$version-bin.zip && \
+    wget https://build.geo-solutions.it/geonode/geoserver/latest/geoserver-$branch.war --no-check-certificate && \
+    unzip geoserver-$version-bin.zip && \
+    mv geoserver-$version geoserver && \
+    rm /geoserver-$version-bin.zip && \
+    rm /geoserver-$version/webapps/geoserver/* -rf && \
+    unzip -o geoserver-$branch.war -d /geoserver/webapps/geoserver/ && \
+    rm /geoserver-$branch.war
+
+# Plugins support
+# Example:
+#   $ docker-compose build --build-arg plugins="csw" geoserver
+RUN for plugin in $plugins; do \
+        wget https://downloads.sourceforge.net/project/geoserver/GeoServer/$version/extensions/geoserver-$version-$plugin-plugin.zip && \
+        unzip -o geoserver-$version-$plugin-plugin.zip -d /geoserver/webapps/geoserver/WEB-INF/lib && \
+        rm geoserver-$version-$plugin-plugin.zip; \
+    done
 
 # Download initial data dir
-RUN wget https://build.geo-solutions.it/geonode/geoserver/latest/data-2.14.x.zip --no-check-certificate
-RUN unzip /data-2.14.x.zip
+RUN wget https://build.geo-solutions.it/geonode/geoserver/latest/data-$branch.zip --no-check-certificate
+RUN unzip /data-$branch.zip
 RUN ls /data
 
-WORKDIR /geoserver-2.14.0/
+WORKDIR /geoserver/
 
 # Add the entrypoint
 ADD docker-entrypoint.sh /docker-entrypoint.sh 
@@ -36,7 +50,7 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 EXPOSE 8080
 
 # Set environnment variables
-ENV GEOSERVER_HOME=/geoserver-2.14.0
+ENV GEOSERVER_HOME=/geoserver
 ENV GEOSERVER_DATA_DIR=/spcgeonode-geodatadir
 
 # Run geoserver

scripts/spcgeonode/geoserver/docker-entrypoint.sh

@@ -55,7 +55,7 @@ fi
 echo "-----------------------------------------------------"
 echo "2. (Re)setting admin account"
 
-ADMIN_ENCRYPTED_PASSWORD=$(/usr/lib/jvm/java-1.8-openjdk/jre/bin/java -classpath /geoserver-2.14.0/webapps/geoserver/WEB-INF/lib/jasypt-1.9.2.jar org.jasypt.intf.cli.JasyptStringDigestCLI digest.sh algorithm=SHA-256 saltSizeBytes=16 iterations=100000 input="$ADMIN_PASSWORD" verbose=0 | tr -d '\n')
+ADMIN_ENCRYPTED_PASSWORD=$(/usr/lib/jvm/java-1.8-openjdk/jre/bin/java -classpath /geoserver/webapps/geoserver/WEB-INF/lib/jasypt-1.9.2.jar org.jasypt.intf.cli.JasyptStringDigestCLI digest.sh algorithm=SHA-256 saltSizeBytes=16 iterations=100000 input="$ADMIN_PASSWORD" verbose=0 | tr -d '\n')
 sed -i -r "s|<user enabled=\".*\" name=\".*\" password=\".*\"/>|<user enabled=\"true\" name=\"$ADMIN_USERNAME\" password=\"digest1:$ADMIN_ENCRYPTED_PASSWORD\"/>|" "/spcgeonode-geodatadir/security/usergroup/default/users.xml"
 # TODO : more selective regexp for this one as there may be several users...
 sed -i -r "s|<userRoles username=\".*\">|<userRoles username=\"$ADMIN_USERNAME\">|" "/spcgeonode-geodatadir/security/role/default/roles.xml"
@@ -71,7 +71,7 @@ echo "3. Wait for PostgreSQL to be ready and initialized"
 # Wait for PostgreSQL
 set +e
 for i in $(seq 60); do
-    psql -h postgres -U postgres -c "ON_ERROR_STOP=1; SELECT client_id FROM oauth2_provider_application" &>/dev/null && break
+    psql "$DATABASE_URL" -c "ON_ERROR_STOP=1; SELECT client_id FROM oauth2_provider_application" &>/dev/null && break
     sleep 1
 done
 if [ $? != 0 ]; then
@@ -90,8 +90,8 @@ echo "4. (Re)setting OAuth2 Configuration"
 # Edit /spcgeonode-geodatadir/security/filter/geonode-oauth2/config.xml
 
 # Getting oauth keys and secrets from the database
-CLIENT_ID=$(psql -h postgres -U postgres -c "SELECT client_id FROM oauth2_provider_application WHERE name='GeoServer'" -t | tr -d '[:space:]')
-CLIENT_SECRET=$(psql -h postgres -U postgres -c "SELECT client_secret FROM oauth2_provider_application WHERE name='GeoServer'" -t | tr -d '[:space:]')
+CLIENT_ID=$(psql "$DATABASE_URL" -c "SELECT client_id FROM oauth2_provider_application WHERE name='GeoServer'" -t | tr -d '[:space:]')
+CLIENT_SECRET=$(psql "$DATABASE_URL" -c "SELECT client_secret FROM oauth2_provider_application WHERE name='GeoServer'" -t | tr -d '[:space:]')
 if [ -z "$CLIENT_ID" ] || [ -z "$CLIENT_SECRET" ]; then
     echo "Could not get OAuth2 ID and SECRET from database. Make sure Postgres container is started and Django has finished it's migrations."
     exit 1

scripts/spcgeonode/pgdumper/Dockerfile

@@ -6,11 +6,14 @@ RUN apk add --no-cache postgresql-client && \
     cp /usr/bin/pg_dump /bin/pg_dump && \
     apk del BUIID_DEPS
 
+# envsubst dependency
+RUN apk add --no-cache gettext
 
 # The entrypoint creates the certificate
-ADD crontab crontab 
-RUN /usr/bin/crontab crontab
-RUN rm crontab
+ADD crontab crontab.envsubst
+ADD docker-entrypoint.sh /
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
 
 # We run cron in foreground to update the certificates
 CMD ["/usr/sbin/crond", "-f"]

scripts/spcgeonode/pgdumper/crontab

@@ -5,4 +5,4 @@
 # │     │     │     │     ┌───────────── day of week (0 - 6) (Sunday to Saturday; 7 is also Sunday on some systems)
 # │     │     │     │     │
 
-  0     0     *     *     *     date && pg_dump -C -h postgres -U postgres postgres > /spcgeonode-pgdumps/latest.pgdump && echo "Dump successful"
+  0     0     *     *     *     date && pg_dump -C "${DATABASE_URL}" > /spcgeonode-pgdumps/latest.pgdump && echo "Dump successful"

scripts/spcgeonode/pgdumper/docker-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+envsubst '\$DATABASE_URL' < crontab.envsubst > crontab
+/usr/bin/crontab crontab
+rm crontab crontab.envsubst
+
+exec "$@"