From bde327c89a35c8c137f4f620ebcc57c0de1c7a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20Wallschl=C3=A4ger?= Date: Tue, 8 Aug 2023 09:21:05 +0200 Subject: [PATCH] [Fixing #8] External PostgreSQL Database support (#47) * fixing issue #8 --- README.md | 3 +- deployment/geonode/templates/_helpers.tpl | 53 +++++++++++- .../templates/geonode/geonode-deploy.yaml | 46 ++++++---- .../geonode/geonode-entrypoint-sh-conf.yaml | 7 +- .../templates/geonode/geonode-env.yaml | 10 +-- .../geonode/geonode-tasks-py-conf.yaml | 28 +++---- .../templates/geoserver/geoserver-deploy.yaml | 5 +- .../templates/geoserver/geoserver-env.yaml | 4 +- .../templates/geoserver/geoserver-svc.yaml | 2 +- .../templates/postgres/geonode-manifest.yaml | 16 ++-- .../postgres-external-geodata-secrets.yaml | 10 +++ .../postgres-external-geonode-secrets.yaml | 10 +++ .../postgres-external-postgres-secrets.yaml | 10 +++ deployment/geonode/values.yaml | 28 ++++--- docs/access-geonode-database-from-outside.md | 84 +++++++++++++++++++ docs/external-database.md | 83 ++++++++++++++++++ minikube-values-external-db.yaml | 54 ++++++++++++ 17 files changed, 389 insertions(+), 64 deletions(-) create mode 100644 deployment/geonode/templates/postgres/postgres-external-geodata-secrets.yaml create mode 100644 deployment/geonode/templates/postgres/postgres-external-geonode-secrets.yaml create mode 100644 deployment/geonode/templates/postgres/postgres-external-postgres-secrets.yaml create mode 100644 docs/access-geonode-database-from-outside.md create mode 100644 docs/external-database.md create mode 100644 minikube-values-external-db.yaml diff --git a/README.md b/README.md index 28ec2c0..df8c025 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,9 @@ If you want to go straight for a production installation follow the [installatio Furhter docs: - [https-ingress](docs/https-ingress.md) -- [access-geonode-database-from-outside-of-kubernetes](docs/external-database-access.md) +- [access-geonode-database-from-outside-of-kubernetes](docs/access-geonode-database-from-outside.md) - [configure-nginx-ingress-body-size-timeout](docs/nginx-ingress-class.md) +- [run-with-external-postgresql-database](docs/external-database.md) Install ------- diff --git a/deployment/geonode/templates/_helpers.tpl b/deployment/geonode/templates/_helpers.tpl index 2441c35..0cfb2f3 100644 --- a/deployment/geonode/templates/_helpers.tpl +++ b/deployment/geonode/templates/_helpers.tpl @@ -1,4 +1,5 @@ + # define pod names (equal service names) {{- define "geoserver_pod_name" -}} {{ .Release.Name }}-{{ .Values.geoserver.pod_name }} @@ -9,7 +10,7 @@ {{- end -}} {{- define "postgres_pod_name" -}} -{{ .Release.Name }}-{{ .Values.postgres.pod_name }} +{{ .Release.Name }}-{{ .Values.postgres.operator_manifest.pod_name }} {{- end -}} {{- define "nginx_pod_name" -}} @@ -17,6 +18,53 @@ {{- end -}} + +# Database definitions + + +{{- define "database_hostname" -}} +{{- if (index .Values "postgres-operator" "enabled") -}} +{{ include "postgres_pod_name" . }} +{{- else if .Values.postgres.external_postgres.enabled -}} +{{- .Values.postgres.external_postgres.hostname -}} +{{- end -}} +{{- end -}} + +{{- define "database_port" -}} +{{- if (index .Values "postgres-operator" "enabled") -}} +5432 +{{- else if .Values.postgres.external_postgres.enabled -}} +{{ .Values.postgres.external_postgres.port }} +{{- end -}} +{{- end -}} + +# secret key reference for the password of user: .Values.postgres.username +{{- define "database_postgres_password_secret_key_ref" -}} +{{- if (index .Values "postgres-operator" "enabled") -}} +"{{ .Values.postgres.username }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do" +{{- else if .Values.postgres.external_postgres.enabled -}} +"{{ .Release.Name }}-postgres-external-secrets" +{{- end -}} +{{- end -}} + +# secret key reference for the password of user: .Values.postgres.geonodedatabase_and_username +{{- define "database_geonode_password_secret_key_ref" -}} +{{- if (index .Values "postgres-operator" "enabled") -}} +"{{ .Values.postgres.geonode_databasename_and_username }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do" +{{- else if .Values.postgres.external_postgres.enabled -}} +"{{ .Release.Name }}-geonode-external-secrets" +{{- end -}} +{{- end -}} + +# secret key reference for the password of user: .Values.postgres.geodatabasename_and_username +{{- define "database_geodata_password_secret_key_ref" -}} +{{- if (index .Values "postgres-operator" "enabled") -}} +"{{ .Values.postgres.geodata_databasename_and_username }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do" +{{- else if .Values.postgres.external_postgres.enabled -}} +"{{ .Release.Name }}-geodata-external-secrets" +{{- end -}} +{{- end -}} + # Volume names {{- define "persistant_volume_name" -}} persistence @@ -24,9 +72,6 @@ persistence # ports and endpoints -{{- define "database_port" -}} -5432 -{{- end -}} {{- define "rabbit_host" -}} {{ .Release.Name }}-rabbitmq:5672 diff --git a/deployment/geonode/templates/geonode/geonode-deploy.yaml b/deployment/geonode/templates/geonode/geonode-deploy.yaml index 2eeb719..7f7041f 100644 --- a/deployment/geonode/templates/geonode/geonode-deploy.yaml +++ b/deployment/geonode/templates/geonode/geonode-deploy.yaml @@ -1,3 +1,18 @@ +# check if external and postgres-operator database backends are active +{{ $postgres_operator := index .Values "postgres-operator" "enabled" }} +{{ $postgres_operator_ui := index .Values "postgres-operator-ui" "enabled"}} +# check if multiple database backends are active +{{ if and .Values.postgres.external_postgres.enabled $postgres_operator }} + {{- fail "Error, two Database backends enabled, check .Values.postgres.external_postgres or .Values.postgres-operator ..." }} +{{ else if and (not .Values.postgres.external_postgres.enabled) (not $postgres_operator) }} + {{- fail "Error, no Database backend is enabled, check .Values.postgres.external_postgres or .Values.postgres-operator ..." }} +{{ end }} +# check if operator ui is activated even postgres-operator is disabled +{{ if and ($postgres_operator_ui) (not $postgres_operator) }} + {{- fail "Error, postgres-operator-ui enabled even postgres-operator ist disabled ..." }} +{{ end }} + + # geonode stateful set apiVersion: apps/v1 kind: StatefulSet @@ -29,7 +44,7 @@ spec: args: - -timeout=60s - -wait - - tcp://{{ include "postgres_pod_name" . }}:{{ include "database_port" .}} + - tcp://{{ include "database_hostname" . }}:{{ include "database_port" . }} - -wait - tcp://{{ include "rabbit_host" .}} @@ -43,7 +58,7 @@ spec: - -c - | # install dockerize... - wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz @@ -78,18 +93,18 @@ spec: env: - name: GEONODE_DATABASE_PASSWORD valueFrom: - secretKeyRef: - name: {{ .Values.postgres.geonodedatabase }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + secretKeyRef: + name: {{ include "database_geonode_password_secret_key_ref" . }} key: password - name: GEONODE_GEODATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.postgres.geodatabasename }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + name: {{ include "database_geodata_password_secret_key_ref" . }} key: password - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.postgres.username }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + name: {{ include "database_postgres_password_secret_key_ref" . }} key: password - name: GEODATABASE_URL value: "postgis://$(GEONODE_GEODATABASE):$(GEONODE_GEODATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_GEODATABASE)" @@ -144,7 +159,7 @@ spec: - -c - | # install dockerize... - wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz @@ -164,35 +179,33 @@ spec: cat /usr/src/geonode/geonode/geonode-k8s-settings.py >> /usr/src/geonode/geonode/settings.py # Setup - touch /usr/src/geonode/invoke.log - dockerize -stdout /usr/src/geonode/invoke.log /usr/src/geonode/entrypoint.sh celery-cmd + touch /var/log/celery.log + dockerize -stdout /var/log/celery.log /usr/src/geonode/entrypoint.sh celery-cmd envFrom: - configMapRef: name: {{ include "geonode_pod_name" . }}-env env: - - name: IS_CELERY - value: 'True' - name: GEONODE_DATABASE_PASSWORD valueFrom: - secretKeyRef: - name: {{ .Values.postgres.geonodedatabase }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + secretKeyRef: + name: {{ include "database_geonode_password_secret_key_ref" . }} key: password - name: GEONODE_GEODATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.postgres.geodatabasename }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + name: {{ include "database_geodata_password_secret_key_ref" . }} key: password - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.postgres.username }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + name: {{ include "database_postgres_password_secret_key_ref" . }} key: password - name: GEODATABASE_URL value: "postgis://$(GEONODE_GEODATABASE):$(GEONODE_GEODATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_GEODATABASE)" - name: DATABASE_URL - value: "postgis://$(GEONODE_DATABASE):$(GEONODE_DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_DATABASE)" + value: "postgis://$(GEONODE_DATABASE):$(GEONODE_DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(GEONODE_DATABASE)" ports: - containerPort: 5555 @@ -265,3 +278,4 @@ spec: # Using an emptyDir to cache compiled statics... it will survive container crashes, but not pod restarts - name: cache-volume emptyDir: {} + diff --git a/deployment/geonode/templates/geonode/geonode-entrypoint-sh-conf.yaml b/deployment/geonode/templates/geonode/geonode-entrypoint-sh-conf.yaml index b2b0242..c54575d 100644 --- a/deployment/geonode/templates/geonode/geonode-entrypoint-sh-conf.yaml +++ b/deployment/geonode/templates/geonode/geonode-entrypoint-sh-conf.yaml @@ -45,8 +45,8 @@ data: echo MONITORING_HOST_NAME=$MONITORING_HOST_NAME echo MONITORING_SERVICE_NAME=$MONITORING_SERVICE_NAME echo MONITORING_DATA_TTL=$MONITORING_DATA_TTL - - invoke waitfordbs + # deactivated until https://github.com/GeoNode/geonode/pull/11340 is merged + #invoke waitfordbs cmd="$@" @@ -61,7 +61,8 @@ data: if [ ${FORCE_REINIT} = "true" ] || [ ${FORCE_REINIT} = "True" ] || [ ! -e "/mnt/volumes/statics/geonode_init.lock" ]; then invoke updategeoip invoke fixtures - invoke monitoringfixture + # currently not implemented in geonode-k8s + # invoke monitoringfixture invoke initialized invoke updateadmin fi diff --git a/deployment/geonode/templates/geonode/geonode-env.yaml b/deployment/geonode/templates/geonode/geonode-env.yaml index 5c9f6fc..74d9055 100644 --- a/deployment/geonode/templates/geonode/geonode-env.yaml +++ b/deployment/geonode/templates/geonode/geonode-env.yaml @@ -191,11 +191,11 @@ data: SENTRY_ENVIRONMENT: {{ .Values.geonode.sentry.environment | quote }} # Database Settings - DATABASE_HOST: "{{ include "postgres_pod_name" . }}" - DATABASE_PORT: "{{ include "database_port" .}}" + DATABASE_HOST: {{ include "database_hostname" . | quote }} + DATABASE_PORT: {{ include "database_port" . | quote }} - POSTGRES_USER: {{ .Values.postgres.username }} - GEONODE_DATABASE: {{ .Values.postgres.geonodedatabase | quote }} - GEONODE_GEODATABASE: {{ .Values.postgres.geodatabasename | quote }} + POSTGRES_USER: {{ .Values.postgres.username | quote }} + GEONODE_DATABASE: {{ .Values.postgres.geonode_databasename_and_username | quote }} + GEONODE_GEODATABASE: {{ .Values.postgres.geodata_databasename_and_username | quote }} GEONODE_DATABASE_SCHEMA: {{ .Values.postgres.schema }} GEONODE_GEODATABASE_SCHEMA: {{ .Values.postgres.schema }} diff --git a/deployment/geonode/templates/geonode/geonode-tasks-py-conf.yaml b/deployment/geonode/templates/geonode/geonode-tasks-py-conf.yaml index 8c95206..9a11f52 100644 --- a/deployment/geonode/templates/geonode/geonode-tasks-py-conf.yaml +++ b/deployment/geonode/templates/geonode/geonode-tasks-py-conf.yaml @@ -84,12 +84,12 @@ data: "monitoring_host_name": os.environ.get("MONITORING_HOST_NAME", "geonode"), "monitoring_service_name": os.environ.get("MONITORING_SERVICE_NAME", "local-geonode"), "monitoring_data_ttl": os.environ.get("MONITORING_DATA_TTL", 7), - "geonode_geodb_passwd": os.environ.get("GEONODE_GEODATABASE_PASSWORD", "geonode_data"), - "default_backend_datastore": os.environ.get("DEFAULT_BACKEND_DATASTORE", "datastore"), - "geonode_db_passwd": os.environ.get("GEONODE_DATABASE_PASSWORD", "geonode"), - "geonode_geodb": os.environ.get("GEONODE_GEODATABASE", "geonode_data"), - "db_url": os.environ.get("DATABASE_URL", "postgis://geonode:geonode@db:5432/geonode"), - "geodb_url": os.environ.get("GEODATABASE_URL", "postgis://geonode:geonode@db:5432/geonode_data"), + "geonode_geodb_passwd": os.environ.get("GEONODE_GEODATABASE_PASSWORD"), + "default_backend_datastore": os.environ.get("DEFAULT_BACKEND_DATASTORE"), + "geonode_db_passwd": os.environ.get("GEONODE_DATABASE_PASSWORD"), + "geonode_geodb": os.environ.get("GEONODE_GEODATABASE"), + "db_url": os.environ.get("DATABASE_URL"), + "geodb_url": os.environ.get("GEODATABASE_URL"), "geonode_db": os.environ.get("GEONODE_DATABASE"), "gs_loc": os.environ.get('GEOSERVER_LOCATION'), "gs_web_ui_loc": os.environ.get("GEOSERVER_WEB_UI_LOCATION"), @@ -664,8 +664,8 @@ data: def _set_geoserver_database_store(): ''' checks if a db store is already created in geoserver, if not create one ''' print("Check if geoserver store is already set up ...") - geoserver_base_url = os.getenv('GEOSERVER_LOCATION', 'geonode-geoserver:8080/geoserver') - geoserver_password = os.getenv('GEOSERVER_ADMIN_PASSWORD', 'geoserver') + geoserver_base_url = os.getenv('GEOSERVER_LOCATION') + geoserver_password = os.getenv('GEOSERVER_ADMIN_PASSWORD') url = '{}rest/workspaces/{}/datastores/'.format(geoserver_base_url,GEOSERVER_WORKSPACE_NAME) headers = {'Content-Type': 'text/xml'} auth = (GEOSERVER_USERNAME, geoserver_password) @@ -681,24 +681,24 @@ data: print("could not get datastore information from geoserver, trying to create ...") print("setup new datastore ...") - db_user = database = os.getenv('GEONODE_GEODATABASE', 'geogeonode') - db_password = os.getenv('GEONODE_GEODATABASE_PASSWORD', 'geogeonode') - db_host = os.getenv('DATABASE_HOST', 'geonode-postgresql') - + db_user = database = os.getenv('GEONODE_GEODATABASE') + db_password = os.getenv('GEONODE_GEODATABASE_PASSWORD') + db_host = os.getenv('DATABASE_HOST') + db_port = os.getenv('DATABASE_PORT') url = '{}rest/workspaces/{}/datastores'.format(geoserver_base_url,GEOSERVER_WORKSPACE_NAME) data = """ {} {} - 5432 + {} {} {} {} postgis - """.format(GEOSERVER_STORE_NAME, db_host, database, db_user, db_password) + """.format(GEOSERVER_STORE_NAME, db_host, db_port, database, db_user, db_password) try: r = requests.post(url, headers=headers, auth=auth,data=data) print("created ...") diff --git a/deployment/geonode/templates/geoserver/geoserver-deploy.yaml b/deployment/geonode/templates/geoserver/geoserver-deploy.yaml index 1042e33..29be5b0 100644 --- a/deployment/geonode/templates/geoserver/geoserver-deploy.yaml +++ b/deployment/geonode/templates/geoserver/geoserver-deploy.yaml @@ -55,7 +55,8 @@ spec: - sh - -c - | - {{`sed -i "s/db:5432/{{DATABASE_HOST}}:5432/g" /templates/geofence/geofence-datasource-ovr.properties.j2`}} + TMP_DB_PORT={{ include "database_port" . }} + {{`sed -i 's/db:5432/{{ DATABASE_HOST }}:$TMP_DB_PORT/g' /templates/geofence/geofence-datasource-ovr.properties.j2`}} /usr/local/tomcat/tmp/entrypoint.sh ports: @@ -70,7 +71,7 @@ spec: - name: GEONODE_GEODATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.postgres.geodatabasename }}.{{ include "postgres_pod_name" . }}.credentials.postgresql.acid.zalan.do + name: {{ include "database_geodata_password_secret_key_ref" . }} key: password volumeMounts: diff --git a/deployment/geonode/templates/geoserver/geoserver-env.yaml b/deployment/geonode/templates/geoserver/geoserver-env.yaml index 0f1b385..bcced59 100644 --- a/deployment/geonode/templates/geoserver/geoserver-env.yaml +++ b/deployment/geonode/templates/geoserver/geoserver-env.yaml @@ -15,9 +15,9 @@ data: GEOSERVER_JAVA_OPTS: "-Djava.awt.headless=true -Xms2G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://geoserver:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine" NGINX_BASE_URL: "{{ include "public_url" . }}/" - DATABASE_HOST: "{{ include "postgres_pod_name" . }}" + DATABASE_HOST: "{{ include "database_hostname" . }}" DATABASE_PORT: "{{ include "database_port" . }}" - GEONODE_GEODATABASE: {{ .Values.postgres.geodatabasename | quote }} + GEONODE_GEODATABASE: {{ .Values.postgres.geonode_databasename_and_username | quote }} GEONODE_GEODATABASE_SCHEMA: {{ .Values.postgres.schema | quote }} GEOSERVER_ADMIN_USER: {{ .Values.geoserver.admin_username | quote }} diff --git a/deployment/geonode/templates/geoserver/geoserver-svc.yaml b/deployment/geonode/templates/geoserver/geoserver-svc.yaml index 9453faa..fe6b779 100644 --- a/deployment/geonode/templates/geoserver/geoserver-svc.yaml +++ b/deployment/geonode/templates/geoserver/geoserver-svc.yaml @@ -9,6 +9,6 @@ spec: org.geonode.instance: "{{ include "geoserver_pod_name" . }}" ports: - targetPort: {{ .Values.geoserver.port }} - port: 8080 + port: {{ .Values.geoserver.port }} name: http type: ClusterIP \ No newline at end of file diff --git a/deployment/geonode/templates/postgres/geonode-manifest.yaml b/deployment/geonode/templates/postgres/geonode-manifest.yaml index d92f9dd..66f12ba 100644 --- a/deployment/geonode/templates/postgres/geonode-manifest.yaml +++ b/deployment/geonode/templates/postgres/geonode-manifest.yaml @@ -1,3 +1,6 @@ + +{{ $postgres_operator := index .Values "postgres-operator" "enabled" }} +{{ if $postgres_operator }} apiVersion: "acid.zalan.do/v1" kind: postgresql metadata: @@ -10,25 +13,25 @@ spec: users: {{ .Values.postgres.username }}: - superuser - {{ .Values.postgres.geonodedatabase }}: + {{ .Values.postgres.geonode_databasename_and_username }}: - superuser - createdb - login - {{ .Values.postgres.geodatabasename }}: + {{ .Values.postgres.geodata_databasename_and_username }}: - superuser - createdb - login databases: - {{ .Values.postgres.geonodedatabase }}: {{ .Values.postgres.geonodedatabase }} - {{ .Values.postgres.geodatabasename }}: {{ .Values.postgres.geodatabasename }} + {{ .Values.postgres.geonode_databasename_and_username }}: {{ .Values.postgres.geonode_databasename_and_username }} + {{ .Values.postgres.geodata_databasename_and_username }}: {{ .Values.postgres.geodata_databasename_and_username }} preparedDatabases: - {{ .Values.postgres.geodatabasename }}: + {{ .Values.postgres.geodata_databasename_and_username }}: schemas: {{ .Values.postgres.schema }}: {} extensions: pg_partman: {{ .Values.postgres.schema }} postgis: {{ .Values.postgres.schema }} - {{ .Values.postgres.geonodedatabase }}: + {{ .Values.postgres.geodata_databasename_and_username }}: schemas: {{ .Values.postgres.schema }}: {} extensions: @@ -36,3 +39,4 @@ spec: postgis: {{ .Values.postgres.schema }} postgresql: version: {{ .Values.postgres.operator_manifest.postgres_version | quote }} +{{ end }} diff --git a/deployment/geonode/templates/postgres/postgres-external-geodata-secrets.yaml b/deployment/geonode/templates/postgres/postgres-external-geodata-secrets.yaml new file mode 100644 index 0000000..c1fbcd5 --- /dev/null +++ b/deployment/geonode/templates/postgres/postgres-external-geodata-secrets.yaml @@ -0,0 +1,10 @@ +{{ if .Values.postgres.external_postgres.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-geodata-external-secrets +type: Opaque +data: + username: {{ .Values.postgres.geodatabasename_and_username | quote }} + password: {{ .Values.postgres.external_postgres.geodata_password | b64enc }} +{{ end }} diff --git a/deployment/geonode/templates/postgres/postgres-external-geonode-secrets.yaml b/deployment/geonode/templates/postgres/postgres-external-geonode-secrets.yaml new file mode 100644 index 0000000..a51593a --- /dev/null +++ b/deployment/geonode/templates/postgres/postgres-external-geonode-secrets.yaml @@ -0,0 +1,10 @@ +{{ if .Values.postgres.external_postgres.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-geonode-external-secrets +type: Opaque +data: + username: {{ .Values.postgres.username | quote }} + password: {{ .Values.postgres.external_postgres.geonode_password | b64enc }} +{{ end }} diff --git a/deployment/geonode/templates/postgres/postgres-external-postgres-secrets.yaml b/deployment/geonode/templates/postgres/postgres-external-postgres-secrets.yaml new file mode 100644 index 0000000..a601325 --- /dev/null +++ b/deployment/geonode/templates/postgres/postgres-external-postgres-secrets.yaml @@ -0,0 +1,10 @@ +{{ if .Values.postgres.external_postgres.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-postgres-external-secrets +type: Opaque +data: + username: {{ .Values.postgres.username | quote }} + password: {{ .Values.postgres.external_postgres.postgres_password | b64enc }} +{{ end }} diff --git a/deployment/geonode/values.yaml b/deployment/geonode/values.yaml index 23b6af1..c46e370 100644 --- a/deployment/geonode/values.yaml +++ b/deployment/geonode/values.yaml @@ -385,29 +385,37 @@ rabbitmq: cpu: "750m" postgres: - # -- pod name for postgres containers == teamID for mainifest - pod_name: postgresql # -- postgres username username: postgres # -- database schema schema: public - # -- geonode database name - geonodedatabase: geonode - # -- geoserver database name - geodatabasename: geogeonode - # database passwords are set randomly - # infos @ https://postgres-operator.readthedocs.io/en/refactoring-sidecars/user/ - # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d - + # -- geonode database name and username + geonode_databasename_and_username: geonode + # -- geoserver database name and username + geodata_databasename_and_username: geodata # -- configuration for postgres operator database manifest operator_manifest: + # -- pod name for postgres containers == teamID for mainifest + pod_name: postgresql # -- Database storage size storageSize: 3Gi # -- number of database instances numberOfInstances: 1 # -- postgres version postgres_version: 15 + # database passwords are set randomly + # infos @ https://postgres-operator.readthedocs.io/en/refactoring-sidecars/user/ + # get password after creation via: kubectl get secret {{ .Release.name }}.{{ .Release.name }}-{{ container_name }}.credentials -o 'jsonpath={.data.password}' | base64 -d + + external_postgres: + enabled: False + hostname: my-external-postgres.com + port: 5432 + postgres_password: postgres + geonode_password: geonode + geodata_password: geogeonode + ######################## # CHART CONFIGURATIONS # diff --git a/docs/access-geonode-database-from-outside.md b/docs/access-geonode-database-from-outside.md new file mode 100644 index 0000000..e517b36 --- /dev/null +++ b/docs/access-geonode-database-from-outside.md @@ -0,0 +1,84 @@ +# Access postgresql Database from Outside + +In some scenarios, e.g. to upload agrovoc or maintain database, its necessary to access the postgresql database from outside of kubernetes. This can be done using the postgres-operator helm chart. Find the documentation at +(https://artifacthub.io/packages/helm/ckotzbauer/postgres-operator?modal=values&path=configLoadBalancer). + +To make the database available from outside we must change the service type from **ClusterIP** to **LoadBalancer**. Therefore we can set the following configuration in our my-values.yaml: +```yaml +postgres-operator: + configLoadBalancer: + db_hosted_zone: geonode.example.org + enable_master_load_balancer: true + external_traffic_policy: Cluster +``` + +Applying this via: +``` +helm upgrade --cleanup-on-fail --install --namespace geonode --create-namespace --values my-values.yaml geonode deployment/geonode +``` + +Will first of all change our service type to **LoadBalancer**. We can double check this with: + +``` +kubectl -n geonode get svc + +# geonode-geonode ClusterIP 10.233.46.30 8000/TCP,8001/TCP 12d +# geonode-geoserver ClusterIP 10.233.22.242 8080/TCP 12d +# geonode-memcached ClusterIP 10.233.56.173 11211/TCP 12d +# geonode-nginx ClusterIP 10.233.31.120 80/TCP 12d +# geonode-postgres-operator ClusterIP 10.233.35.162 8080/TCP 12d +# geonode-postgres-operator-ui ClusterIP 10.233.48.133 80/TCP 12d +# geonode-postgresql LoadBalancer 10.233.23.191 5432:31360/TCP 21d +# geonode-postgresql-config ClusterIP None 21d +# geonode-postgresql-repl ClusterIP 10.233.2.166 5432/TCP 21d +# geonode-rabbitmq ClusterIP 10.233.52.33 5672/TCP,4369/TCP,25672/TCP,15672/TCP 12d +# geonode-rabbitmq-headless ClusterIP None 4369/TCP,5672/TCP,25672/TCP,15672/TCP 12d +``` + +Here you can see the **geonode-postgresql** service has now the service type **LoadBalancer**. Furhter we found the NodePort to be 31360. You can get detailed information about this service via: + +``` +kubectl -n geonode describe svc geonode-postgresql + +# Name: geonode-postgresql +# Namespace: geonode +# Labels: application=spilo +# cluster-name=geonode-postgresql +# spilo-role=master +# team=geonode +# Annotations: external-dns.alpha.kubernetes.io/hostname: postgresql.geonode.geonode.example.org +# service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: 3600 +# Selector: +# Type: LoadBalancer +# IP Family Policy: SingleStack +# IP Families: IPv4 +# IP: 10.233.23.191 +# IPs: 10.233.23.191 +# Port: postgresql 5432/TCP +# TargetPort: 5432/TCP +# NodePort: postgresql 31360/TCP +# Endpoints: 10.233.99.42:5432 +# Session Affinity: None +# External Traffic Policy: Cluster +# LoadBalancer Source Ranges: 127.0.0.1/32 +# Events: +# Type Reason Age From Message +# ---- ------ ---- ---- ------- +# Normal Type 20m service-controller ClusterIP -> LoadBalancer +``` + +Here you can find again the NodePort. Also the external dns name is set here **Annotations** as **postgresql.geonode.geonode.example.org**. +Before you can connect to the database check the postgres operator secrets for the geonode user password via: +``` +# usernames might be different regarding your .Values.postgres.geonode.{username|geonodedatabase|geodatabasename} configuration +# get geonode user password +kubectl -n geonode get secret geonode.geonode-postgresql.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d +# get postgres user password +kubectl -n geonode get secret postgres.geonode-postgresql.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d +# get geogeonode user password +``` + +So now you can connect to the database via: +```bash +psql -h postgresql.geonode.geonode.example.org -p 31360 -U geonode +``` \ No newline at end of file diff --git a/docs/external-database.md b/docs/external-database.md new file mode 100644 index 0000000..af87ab6 --- /dev/null +++ b/docs/external-database.md @@ -0,0 +1,83 @@ +# Deploying using an external postgresql Database + +Geonode-k8s supports an external postgresql database. This database **requires to have postgis extension installed**. If you gonna use an external [postgres-operator](https://github.com/zalando/postgres-operator), here is a template based on the one used inside this helm chart: + +``` +# Source: geonode-k8s/templates/postgres/geonode-manifest.yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: "geonode-postgresql" +spec: + teamId: dis + volume: + size: 10Gi + numberOfInstances: 2 + users: + postgres: + - superuser + geonode: + - superuser + - createdb + - login + geodata: + - superuser + - createdb + - login + databases: + geonode: geonode + geodata: geodata + preparedDatabases: + geodata: + schemas: + public: {} + extensions: + pg_partman: public + postgis: public + geodata: + schemas: + public: {} + extensions: + pg_partman: public + postgis: public + postgresql: + version: "15" +``` +Get passwords from postgres-operator like: +``` +kubectl get secret postgres.geonode-postgresql.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d +kubectl get secret geonode.geonode-postgresql.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d +kubectl get secret geodata.geonode-postgresql.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d +``` + +But also any other postgis database can be used. For GeoNode 4.1 it is required to use postgresql version 15. + +Now you have to configure your values.yaml to use this external database. You can use `minikube-values-external-db.yaml` or the example below in your values.yaml: + +``` +postgres: + username: postgres + geonode_databasename_and_username: geonode + geodata_databasename_and_username: geodata + + external_postgres: + enabled: True + hostname: my-external-postgres.com + port: 5432 + postgres_password: + geonode_password: + geodata_password: + +postgres-operator: + enabled: False +``` + +To deploy run helm and give passwords as helm arguments like: +``` +export GEONODE_K8S_POSTGRES_PASSWORD="password" +export GEONODE_K8S_GEONODE_PASSOWRD="password" +export GEONODE_K8S_GEODATA_PASSWORD="password" +helm upgrade --cleanup-on-fail --install --namespace geonode --create-namespace --values minikube-values-external-db.yaml --set postgres.external_postgres.postgres_password=${GEONODE_K8S_POSTGRES_PASSWORD} --set postgres.external_postgres.geonode_password=${GEONODE_K8S_GEONODE_PASSOWRD} --set postgres.external_postgres.geodata_password=${GEONODE_K8S_GEODATA_PASSWORD} geonode deployment/geonode +``` + +If run on minikube follow the original [minikube docs](minikube-installation.md) for accessing the geonode installation through `minikube tunnel`. \ No newline at end of file diff --git a/minikube-values-external-db.yaml b/minikube-values-external-db.yaml new file mode 100644 index 0000000..df0edc1 --- /dev/null +++ b/minikube-values-external-db.yaml @@ -0,0 +1,54 @@ +global: + storageClass: standard + +geonode: + replicaCount: 1 # not working yet + + general: + debug: True + debug_static: True + + persistant: + storageSize: 2Gi + + ingress: + enabled: False + externalScheme: http + externalDomain: geonode + externalPort: 80 + + superUser: + password: geonode + + mail: + enabled: False + + ldap: + enabled: False + + register: + open_signup: True + email_verification: "False" + authentication_method: username_email + +rabbitmq: + replicaCount: 1 + auth: + username: rabbituser + password: rabbit_password + +postgres: + username: postgres + geonode_databasename_and_username: geonode + geodata_databasename_and_username: geodata + + external_postgres: + enabled: True + hostname: "external-postgres.com" + port: 5432 + postgres_password: + geonode_password: + geodata_password: +postgres-operator: + enabled: False +