Adding CI/CD container "jenkins"

.env

@@ -59,6 +59,13 @@ DEFAULT_BACKEND_UPLOADER=geonode.importer
 TIME_ENABLED=True
 MOSAIC_ENABLED=False
 
+# #################
+# Jenkins
+# CI/CD Server
+# #################
+JENKINS_HTTP_PORT=9080
+JENKINS_HTTPS_PORT=9443
+
 # #################
 # nginx
 # HTTPD Server

docker-compose.override.yml

@@ -8,3 +8,8 @@ services:
     # old copy that docker added when creating the image
     volumes:
       - '.:/usr/src/{{project_name}}'
+
+  jenkins:
+    # Link to local volume in the HOST
+    volumes:
+      - '/data/backup_restore:/backup_restore'

docker-compose.yml

@@ -174,7 +174,25 @@ services:
     volumes:
       - geoserver-data-dir:/geoserver_data/data
 
+  jenkins:
+    image: jenkins/jenkins:lts
+    # image: istresearch/jenkins:latest
+    container_name: jenkins4${COMPOSE_PROJECT_NAME}
+    user: jenkins
+    ports:
+      - '${JENKINS_HTTP_PORT}:${JENKINS_HTTP_PORT}'
+      - '${JENKINS_HTTPS_PORT}:8443'
+      - '50000:50000'
+    volumes:
+      - jenkins_data:/var/jenkins_home
+      - backup-restore:/backup_restore
+      # - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      - 'JENKINS_OPTS=--httpPort=${JENKINS_HTTP_PORT} --httpsPort=${JENKINS_HTTPS_PORT} --prefix=/jenkins'
+
 volumes:
+  jenkins_data:
+    driver: local
   statics:
     name: ${COMPOSE_PROJECT_NAME}-statics
   nginx-confd:

docker/jenkins/.env

@@ -0,0 +1,3 @@
+COMPOSE_PROJECT_NAME={{project_name}}_jenkins
+JENKINS_HTTP_PORT=9080
+JENKINS_HTTPS_PORT=9443

docker/jenkins/README.md

@@ -0,0 +1,94 @@
+Docker Composer Setup
+=====================
+
+> How to easily get Jenkins with the Blue Ocean UI up and running on your computer using `docker` and `docker-compose`
+
+## How to use:
+
+0. - Install [Docker and Docker Compose](https://www.docker.com/community-edition) for your OS. 
+
+   + For Windows OS without Hyper-V Capability
+     + Install Docker toolbox (https://docs.docker.com/toolbox/toolbox_install_windows/)
+     + Once installed you might need to remove the line `"credsStore": "wincred"` from `~/.docker/config.json`
+     + Execute all command using `Docker Quickstart Terminal`
+
+1. In your directory, copy `.env.sample` to `.env` and customize as needed.  In particular, change `JENKINS_HTTP_PORT` and `JENKINS_HTTPS_PORT` to a high port (e.g. 18443) on Minikube.
+
+2. Run docker-compose to bring up Jenkins
+
+```sh
+$ docker-compose up -d
+```
+
+3. Get your initial admin password
+
+```sh
+$ docker-compose exec jenkins cat /var/jenkins_home/secrets/initialAdminPassword
+```
+
+4. Open Jenkins in your browser by going to either http://0.0.0.0:8080/ or https://0.0.0.0:8443/ (or whatever your Docker Machine IP is).
+
+5. Enter your initial admin password when the setup wizard asks you to.
+
+![Start the setup wizard by entering the intial admin password](images/jenkins-unlock.png)
+
+6. Choose `Install suggested plugins`
+
+![Jenkins Install Options](images/jenkins-install.png)
+
+![Jenkins is Installing](images/jenkins-setup.png)
+
+7. Enter your admin user details and click `Save and Finish`
+
+![Jenkins Admin User Setup](images/jenkins-admin-user.png)
+
+8. Click `Open Blue Ocean` from the left hand menu
+
+![Blue Ocean](images/jenkins-blue-ocean.png)
+
+> Congratulations! You may now use Jenkins with the Blue Ocean UI and the Jenkins Pipelines of 2.0
+
+## Troubleshooting
+
+#### Mounted volumes are empty
+
+This guide is for:
+- Windows users that did not meet the system requirements of __Docker for Windows__
+(You may check out the system requirements [here](https://docs.docker.com/docker-for-windows/install/#what-to-know-before-you-install))
+- Uses __Docker Toolbox__
+- Uses __Virtualbox__ for virtualization
+
+_Note: We will assume that your current working directory is at `D:\Projects\Jenkins`_
+_Note: All commands are executed inside the __Docker Quickstart Terminal___
+
+1. In your virtual machine, create a shared folder that has access to your working directory
+You may share `D:\Projects` instead of `D:\Projects\Jenkins`.
+Everything under `D:\Projects` will be accessible in the vm.
+
+    - Open Virtualbox
+    - Open the vm's settings
+        - Your vm's name is probably __default__
+        - You can see which machine is currently active by running `docker-machine ls`
+        The active machine should have an asterisk (`*`) on the `ACTIVE` column.
+    - Under _Shared Folders_ add a new entry
+        - If folder path is `D:\Projects`, folder name should be `d/Projects`
+        - Tick `Auto-mount`
+        - Tick `Make Permanent`
+
+![vm settings](images/vm_settings.png)
+
+sometimes, `Make Permanent` won't be an option. Don't bother looking for that if so, it'll still be fine. Auto-mount is what we want.
+
+2. If your vm is running, Restart your vm by executing `docker-machine restart`
+If your vm is stopped, you are either not executing these inside the _quickstart terminal_
+or you need to execute `docker-machine start <vm-name-here>`
+
+3. Verify the shared folder by logging in to the vm
+    - run `docker-machine ssh`
+    - run `ls /d/Projects`
+
+Now your containers should have access to your current folder.
+
+### NOTES
+- When setting up the `Folder Name` in the VM settings, assure that it's a valid linux path and must be relative to `/` (root) directory (eg `e/projects`).
+- The directory that contains `jenkins` **must not contain** any whitespaces or any characters that must be escaped. (e.g `F19 Source Code`, `Source\ Code`) The reason is that it'll serve as a `Folder Name` in your VM settings, and you cannot click OK when you do supply such names.

docker/jenkins/docker-compose.override.yml

@@ -0,0 +1,6 @@
+version: '2.2'
+services:
+
+  jenkins:
+    volumes:
+      - '/data/backup_restore:/backup_restore'

docker/jenkins/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '2.2'
+services:
+  jenkins:
+    image: 'jenkins/jenkins:lts'
+    labels:
+      kompose.service.type: nodeport
+    ports:
+      - '${JENKINS_HTTP_PORT}:8080'
+      - '${JENKINS_HTTPS_PORT}:8443'
+      - '50000:50000'
+    volumes:
+      - 'jenkins_data:/var/jenkins_home'
+      - backup-restore:/backup_restore
+    env_file:
+      - .env
+    environment:
+      - 'JENKINS_OPTS=--httpPort=8080 --httpsPort=8443'
+
+volumes:
+  jenkins_data:
+    driver: local
+  backup-restore:
+    name: ${COMPOSE_PROJECT_NAME}-backup-restore

docker/nginx/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache openssl inotify-tools
 WORKDIR /etc/nginx/
 
 ADD nginx.conf.envsubst nginx.https.available.conf.envsubst ./
-ADD geonode.conf ./sites-enabled/
+ADD geonode.conf.envsubst ./sites-enabled/
 
 ADD docker-autoreload.sh docker-entrypoint.sh /
 ENTRYPOINT ["/docker-entrypoint.sh"]

docker/nginx/docker-entrypoint.sh

@@ -39,10 +39,14 @@ fi
 if [ -z "${HTTPS_PORT}" ]; then
         HTTPS_PORT=443
 fi
+if [ -z "${JENKINS_HTTP_PORT}" ]; then
+        JENKINS_HTTP_PORT=9080
+fi
 
 echo "Replacing environement variables"
 envsubst '\$HTTP_PORT \$HTTPS_PORT \$HTTP_HOST \$HTTPS_HOST \$RESOLVER' < /etc/nginx/nginx.conf.envsubst > /etc/nginx/nginx.conf
 envsubst '\$HTTP_PORT \$HTTPS_PORT \$HTTP_HOST \$HTTPS_HOST \$RESOLVER' < /etc/nginx/nginx.https.available.conf.envsubst > /etc/nginx/nginx.https.available.conf
+envsubst '\$JENKINS_HTTP_PORT' < /etc/nginx/sites-enabled/geonode.conf.envsubst > /etc/nginx/sites-enabled/geonode.conf
 
 echo "Enabling or not https configuration"
 if [ -z "${HTTPS_HOST}" ]; then 

docker/nginx/geonode.conf → docker/nginx/geonode.conf.envsubst

@@ -48,6 +48,21 @@ location /geoserver {
     proxy_pass http://$upstream;
 }
 
+# Jenkins
+location /jenkins {
+
+    # Using a variable is a trick to let Nginx start even if upstream host is not up yet
+    # (see https://sandro-keil.de/blog/2017/07/24/let-nginx-start-if-upstream-host-is-unavailable-or-down/)
+    set $upstream jenkins:$JENKINS_HTTP_PORT;
+
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
+    proxy_pass http://$upstream;
+}
+
 # GeoNode
 location /static/ {
   alias /mnt/volumes/statics/static/;

uwsgi.ini

@@ -18,13 +18,13 @@ need-app = true
 touch-reload = /usr/src/{{project_name}}/{{project_name}}/wsgi.py
 buffer-size = 32768
 
-harakiri = 60                        ; forcefully kill workers after 60 seconds
+harakiri = 600                       ; forcefully kill workers after 600 seconds
 py-callos-afterfork = true           ; allow workers to trap signals
 
 max-requests = 1000                  ; Restart workers after this many requests
 max-worker-lifetime = 3600           ; Restart workers after this many seconds
 reload-on-rss = 2048                 ; Restart workers after this much resident memory
-worker-reload-mercy = 60             ; How long to wait before forcefully killing workers
+worker-reload-mercy = 600            ; How long to wait before forcefully killing workers
 
 cheaper-algo = busyness
 processes = 128                      ; Maximum number of workers allowed
@@ -39,4 +39,6 @@ cheaper-busyness-max = 70            ; Above this threshold, spawn new workers
 cheaper-busyness-backlog-alert = 16  ; Spawn emergency workers if more than this many requests are waiting in the queue
 cheaper-busyness-backlog-step = 2    ; How many emergency workers to create if there are too many requests in the queue
 
-# cron = -1 -1 -1 -1 -1 /usr/local/bin/python /usr/src/{{project_name}}/manage.py collect_metrics -n
+# cron = -1 -1 -1 -1 -1 sh -c '/usr/src/{{project_name}}/manage.sh collect_metrics -n -t xml';
+cron = 0 0 -1 -1 -1 sh -c 'find /backup_restore/ -type f -mtime +30 -exec rm -f {} \;'
+cron = 0 0 -1 -1 -1 sh -c 'find /backup_restore/ -type d -ctime +30 -exec rm -rf {} \;'