Replace vagrant-based testing with docker-based testing

.env.example

@@ -0,0 +1,75 @@
+# Host Ports
+CKAN_PORT_HOST=5000
+NGINX_PORT_HOST=81
+NGINX_SSLPORT_HOST=8443
+
+# CKAN databases
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+POSTGRES_HOST=db
+CKAN_DB_USER=ckandbuser
+CKAN_DB_PASSWORD=ckandbpassword
+CKAN_DB=ckandb
+DATASTORE_READONLY_USER=datastore_ro
+DATASTORE_READONLY_PASSWORD=datastore
+DATASTORE_DB=datastore
+CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb
+CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
+
+# Test database connections
+TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
+TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
+
+# Dev settings
+USE_HTTPS_FOR_DEV=false
+
+# CKAN core
+CKAN_VERSION=2.10.0
+CKAN_SITE_ID=default
+CKAN_SITE_URL=https://localhost:8443
+CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
+# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
+CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
+CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
+CKAN_SYSADMIN_NAME=ckan_admin
+CKAN_SYSADMIN_PASSWORD=test1234
+[email protected]
+CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+CKAN_MAX_UPLOAD_SIZE_MB=100
+TZ=UTC
+
+# Solr
+SOLR_IMAGE_VERSION=2.10-solr9
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
+
+# Redis
+REDIS_VERSION=6
+CKAN_REDIS_URL=redis://redis:6379/1
+TEST_CKAN_REDIS_URL=redis://redis:6379/1
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.20
+CKAN_DATAPUSHER_URL=http://datapusher:8800
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
+
+# NGINX
+NGINX_PORT=80
+NGINX_SSLPORT=443
+
+# Extensions
+CKAN__PLUGINS="image_view text_view datatables_view datastore datapusher envvars"
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=redis
+CKAN__HARVEST__MQ__PORT=6379
+CKAN__HARVEST__MQ__REDIS_DB=1
+
+CKAN___LICENSES_GROUP_URL=file:///srv/app/.local/lib/python3.10/site-packages/ckanext/dcor_schemas/licenses.json

.github/workflows/check.yml

@@ -8,42 +8,40 @@ on:
 concurrency: general_workflow
 
 jobs:
-  checks:
-    runs-on: macos-12
+  test:
+    runs-on: ubuntu-latest
+
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.8"
-    - name: Install flake8 dependencies
-      run: |
-        # prerequisites
-        python -m pip install --upgrade pip wheel
-        python -m pip install flake8
-    - name: Lint with flake8
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Create .env file
+      run: cp .env.example .env
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v2
+
+    - name: Install Docker Compose
       run: |
-        flake8 --exclude _version.py .
-    - name: Cache vagrant images
-      uses: actions/cache@v3
-      env:
-        cache-name: vagrant-image
+        sudo curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        sudo chmod +x /usr/local/bin/docker-compose
+        docker-compose --version  # Verify installation
+
+    - name: Start Docker Compose services
+      run: docker-compose -f docker-compose.ci.yml up -d
+
+    - name: Wait for CKAN to be healthy
+      run: docker-compose -f docker-compose.ci.yml exec dcor-test bash -c "while ! wget -qO- http://localhost:5000; do sleep 5; done"
+
+    - name: Run Tests
+      run: bash ./docker-run-tests.sh
+
+    - name: Upload Coverage to GitHub
+      uses: actions/upload-artifact@v4
       with:
-        path: ~/.vagrant.d
-        key: macos-build-${{ env.cache-name }}-${{ hashFiles('Vagrantfile') }}
-        restore-keys: |
-          macos-build-${{ env.cache-name }}-
-    - name: save environment variables for codecov in VM
-      run: |
-        env | grep GITHUB > environment.txt
-        cat environment.txt
-    - name: Get the latest box and remove old boxes from the GH Actions cache
-      run: |
-        vagrant box update
-        vagrant box prune
-    - name: Setup vagrant VM
-      run: |
-        vagrant up
-    - name: Run tests in vagrant VM
-      run: |
-        vagrant ssh -- sudo bash /testing/vagrant-run-tests.sh
+        name: coverage-report
+        path: coverage.xml
+
+    - name: Bring Down Docker Compose
+      if: always()
+      run: docker-compose -f docker-compose.ci.yml down

CHANGELOG

@@ -1,3 +1,5 @@
+0.8.3
+ - enh: implement docker based testing
 0.8.2
  - setup: fix ckanext namespace
 0.8.1

docker-compose.ci.yml

@@ -0,0 +1,101 @@
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data:
+  pip_cache:
+  site_packages:
+  vscode_server:
+  minio_data:
+
+services:
+  dcor-test:
+    # Use image from Raghava's dockerHub
+    image: raghavaalajangi95/dcor-ckan-docker-ckan:latest
+    env_file:
+      - .env
+    links:
+      - db
+      - solr
+      - redis
+      - minio
+    # hostname is defined as a docker resource in `dcor_control`
+    hostname: dcor-dev-docker
+    ports:
+      - "0.0.0.0:${CKAN_PORT_HOST}:5000"
+    user: root # This will avoid permission errors
+    volumes:
+      - ckan_storage:/var/lib/ckan
+      # Map current GitHub runner working dir as src_extensions dir within 
+      # the container (this will allow us to access the ckanext-dcor_theme 
+      # directly inside src_extensions)
+      - ./:/srv/app/src_extensions
+      - pip_cache:/root/.cache/pip
+      - site_packages:/usr/lib/python3.10/site-packages
+      - vscode_server:/root/.vscode-server
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+
+  datapusher:
+    image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"]
+
+  db:
+    # Use image from Raghava's dockerHub
+    image: raghavaalajangi95/dcor-ckan-docker-postgresql:latest
+    environment:
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - CKAN_DB_USER
+      - CKAN_DB_PASSWORD
+      - CKAN_DB
+      - DATASTORE_READONLY_USER
+      - DATASTORE_READONLY_PASSWORD
+      - DATASTORE_DB
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+
+  solr:
+    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    volumes:
+      - solr_data:/var/solr
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
+
+  redis:
+    image: redis:${REDIS_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "-e", "QUIT"]
+
+  minio:
+    image: minio/minio:latest
+    restart: unless-stopped
+    environment:
+      - MINIO_ROOT_USER
+      - MINIO_ROOT_PASSWORD
+    command: minio server --console-address ":9001" /data
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio_data:/data
+
+  mc:
+    image: minio/mc:RELEASE.2020-08-08T02-33-58Z
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc config host rm local;
+      /usr/bin/mc config host add --api s3v4 local http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb local/ckan/;
+      /usr/bin/mc policy set download local/ckan/storage;
+      "

docker-run-tests.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Fail immediately if any command fails
+set -e
+
+EXTENSION_NAME="ckanext-dcor_theme"  # Change this to your extension's name
+CKAN_CONTAINER="${EXTENSION_NAME}-dcor-test-1" # CKAN container name
+EXTENSION_PATH="/srv/app/src_extensions/"
+
+# Create venv and install dependencies as a root user inside the container
+docker exec -u root ${CKAN_CONTAINER} bash -c "
+  cd ${EXTENSION_PATH};
+
+  # Create a venv with systme site packages
+  python3 -m venv --system-site-packages venv;
+  source venv/bin/activate;
+  pip install --upgrade pip wheel;
+
+  # Update dcor libraries
+  dcor update --yes;
+
+  # Install ckanext-dcor_theme and its test requirements
+  pip install .;
+  pip install -r ./ckanext/dcor_theme/tests/requirements.txt;
+
+  # Change ownership so that 'ckan' user can use the virtual environment
+  chown -R ckan:ckan-sys ${EXTENSION_PATH}/venv
+"
+
+# Run tests on GitHub runner where container gets permissions from.
+echo "Running tests in the virtual environment..."
+docker exec ${CKAN_CONTAINER} bash -c "
+  cd ${EXTENSION_PATH};
+  source venv/bin/activate;
+
+  # Run coverage
+  coverage run --source=ckanext.dcor_theme --omit=*tests* -m pytest -p no:warnings ckanext;
+
+  # Generate the XML report
+  coverage xml
+"
+
+echo "Tests completed!"