bayanat v1.31

.env-sample

@@ -37,41 +37,10 @@ SECURITY_TWO_FACTOR=True
 SECURITY_TWO_FACTOR_RESCUE_MAIL=''
 SECURITY_TWO_FACTOR_AUTHENTICATOR_VALIDITY=90
 
-# S3
-AWS_ACCESS_KEY_ID=''
-AWS_SECRET_ACCESS_KEY=''
-S3_BUCKET=''
-AWS_REGION='us-east-1'
-
-# Recaptcha
-RECAPTCHA_ENABLED=False
-RECAPTCHA_PUBLIC_KEY=''
-RECAPTCHA_PRIVATE_KEY=''
-
-FILESYSTEM_LOCAL=True
-ETL_TOOL=True
-ETL_PATH_IMPORT=False
-SHEET_IMPORT=True
-MISSING_PERSONS=False
-
-# MAPS
-MAPS_API_ENDPOINT='https://tile.openstreetmap.org/{z}/{x}/{y}.png'
-
-#Google Maps API
-GOOGLE_MAPS_API_KEY=''
-
-# Google Oauth
-GOOGLE_CLIENT_ID=''
-GOOGLE_CLIENT_SECRET=''
-
-# Deduplication
-DEDUP_TOOL=False
-DEDUP_LOW_DISTANCE=0.3
-DEDUP_MAX_DISTANCE=0.5
-DEDUP_BATCH_SIZE=30
-
 # Secure cookies
 # Requires SSL to work properly
 # Should be set to true in prod envs
 # Can be switched off in dev envs
-SECURE_COOKIES=True
+SECURE_COOKIES=True
+
+TESSERACT_CMD="/opt/homebrew/bin/tesseract"

.github/workflows/run-tests.yml

@@ -0,0 +1,20 @@
+# test compose setup
+name: Bayanat tests
+on: [push]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: docker:latest
+        options: --privileged
+        ports:
+          - 80:80
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Create .env file
+        run: |
+          ./gen-env.sh -d -e .env.test
+      - name: Bayanat Tests
+        run: docker compose -f docker-compose-prebuilt-test.yml --env-file .env.test up --build --exit-code-from bayanat

.gitignore

@@ -12,7 +12,7 @@ flask_session
 .mypy_cache/
 .history
 celerybeat-schedule.db
-.env
+.env*
 enferno/media/*
 *.log
 *.csv

CONTRIBUTING.md

@@ -82,3 +82,26 @@ The configuration for each tool is as follows:
 
 - **Black:** Configured via `pyproject.toml`.
 - **Prettier:** Configured via `.prettierrc`.
+
+# Tests
+
+Bayanat comes with e2e tests using pytest and pydantic models. To run the tests, install the dependencies with
+```bash
+pip install -r dev-requirements.txt
+```
+
+Bayanat tests require a separate test database to be setup before running any tests.
+
+After setting up the production database, follow the instructions below for creating a test database
+
+```bash
+sudo -u bayanat createdb bayanat_test
+sudo -u postgres psql -d bayanat_test -c 'CREATE EXTENSION if not exists pg_trgm; CREATE EXTENSION if not exists postgis;'
+```
+
+After creating the test db, you can run the existing tests with
+```bash
+pytest
+```
+
+Tests define `pydantic` models to ensure the backend responses and frontend requests conform to expected database schema.

README.md

@@ -12,11 +12,7 @@ Installation guidelines and user manual are available at [docs.bayanat.org](http
 
 Localization
 ------------
-Bayanat currently has translations for the following languages:
-- Arabic
-- Ukrainian
-
-You can help by translating Bayanat to your language. `messages.pot` file is the translation template for Bayanat.
+Check Bayanat's [project](https://poeditor.com/join/project/XRamVw2AD0) on POEditor to check on current available languages and to help translating Bayanat to your language.
 
 Live Demo
 ---------

config.sample.json

@@ -60,18 +60,22 @@
   "LANGUAGES": [
     "en",
     "ar",
-    "uk"
+    "uk",
+    "fr",
+    "fa",
+    "es",
+    "ru"
   ],
   "MAPS_API_ENDPOINT": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
   "MEDIA_ALLOWED_EXTENSIONS": [
-    ".mp4",
-    ".webm",
-    ".jpg",
-    ".gif",
-    ".png",
-    ".pdf",
-    ".doc",
-    ".txt"
+    "mp4",
+    "webm",
+    "jpg",
+    "gif",
+    "png",
+    "pdf",
+    "doc",
+    "txt"
   ],
   "MISSING_PERSONS": false,
   "OCR_ENABLED": false,
@@ -92,9 +96,9 @@
   "SECURITY_FRESHNESS": 21600,
   "SECURITY_FRESHNESS_GRACE_PERIOD": 1800,
   "SHEETS_ALLOWED_EXTENSIONS": [
-    ".csv",
-    ".xls",
-    ".xlsx"
+    "csv",
+    "xls",
+    "xlsx"
   ],
   "SHEET_IMPORT": true,
   "TESSERACT_CMD": "/opt/homebrew/bin/tesseract"

dev-requirements.txt

@@ -1,3 +1,8 @@
 -r requirements.txt
+pytest==7.4.4
+pydantic==1.10.13
+flask_pydantic==0.11.0
+SQLAlchemy-Utils==0.41.1
 pre-commit==3.5.0
 black==23.11.0
+factory-boy==3.3.0

docker-compose-dev.yml

@@ -0,0 +1,61 @@
+version: '3'
+
+services:
+  postgres:
+    container_name: postgres
+    image: 'postgis/postgis:15-3.3'
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB}
+    volumes:
+      - 'postgres_dev_data:/var/lib/postgresql/data'
+    expose:
+      - '5432'
+    healthcheck:
+      test: "pg_isready -d ${POSTGRES_DB} -U ${POSTGRES_USER}"
+      interval: 3s
+      retries: 10
+
+  redis:
+    container_name: redis
+    image: 'redis:latest'
+    expose:
+      - '6379'
+    command: redis-server --requirepass '${REDIS_PASSWORD}'
+    volumes:
+      - 'redis_dev_data:/var/lib/redis/data:rw'
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
+      interval: 3s
+      retries: 10
+
+  bayanat:
+    container_name: bayanat
+    image: bayanat/bayanat:dev
+    build:
+      context: .
+      dockerfile: ./flask/Dockerfile.dev
+      args:
+        - ROLE=flask
+        - ENV_FILE=${ENV_FILE:-.env.dev}
+    volumes:
+      - 'bayanat_dev_media:/app/enferno/media/:rw'
+      - 'bayanat_dev_imports:/app/enferno/imports:rw'
+      - '${PWD}/config.json:/app/config.json:rw'
+      - '${PWD}/reload.ini:/app/reload.ini:rw'
+      - '${PWD}/${ENV_FILE:-.env.dev}:/app/.env:ro'
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    ports:
+      - '127.0.0.1:5000:5000'
+    restart: always
+
+volumes:
+  redis_dev_data:
+  postgres_dev_data:
+  bayanat_dev_media:
+  bayanat_dev_imports:

docker-compose-prebuilt-dev.yml

@@ -0,0 +1,68 @@
+version: '3'
+
+services:
+  postgres:
+    container_name: postgres
+    image: 'postgis/postgis:15-3.3'
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB}
+    volumes:
+      - 'postgres_dev_data:/var/lib/postgresql/data:rw'
+    expose:
+      - '5432'
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    tmpfs:
+      - /var/run/postgresql
+    healthcheck:
+      test: "pg_isready -d ${POSTGRES_DB} -U ${POSTGRES_USER}"
+      interval: 3s
+      retries: 10
+
+  redis:
+    container_name: redis
+    image: 'redis:latest'
+    expose:
+      - '6379'
+    command: redis-server --requirepass '${REDIS_PASSWORD}'
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    volumes:
+      - 'redis_dev_data:/var/lib/redis/data:rw'
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
+      interval: 3s
+      retries: 10
+
+  bayanat:
+    container_name: bayanat
+    image: 'bayanat/bayanat:dev'
+    volumes:
+      - 'bayanat_dev_media:/app/enferno/media:rw'
+      - 'bayanat_dev_imports:/app/enferno/imports:rw'
+      - '${PWD}/config.json:/app/config.json:rw'
+      - '${PWD}/reload.ini:/app/reload.ini:rw'
+      - '${PWD}/${ENV_FILE:-.env.dev}:/app/.env:ro'
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    read_only: true
+    tmpfs:
+      - /tmp
+    security_opt:
+      - no-new-privileges:true
+    ports:
+      - '127.0.0.1:5000:5000'
+    restart: always
+
+volumes:
+  redis_dev_data:
+  postgres_dev_data:
+  bayanat_dev_media:
+  bayanat_dev_imports:

docker-compose-prebuilt-test.yml

@@ -0,0 +1,63 @@
+version: '3'
+
+services:
+  postgres:
+    container_name: postgres
+    image: 'postgis/postgis:15-3.3'
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=bayanat_test
+    volumes:
+      - 'postgres_test_data:/var/lib/postgresql/data:rw'
+    expose:
+      - '5432'
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    tmpfs:
+      - /var/run/postgresql
+    healthcheck:
+      test: "pg_isready -d bayanat_test -U ${POSTGRES_USER}"
+      interval: 3s
+      retries: 10
+
+  redis:
+    container_name: redis
+    image: 'redis:latest'
+    expose:
+      - '6379'
+    command: redis-server --requirepass '${REDIS_PASSWORD}'
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    volumes:
+      - 'redis_test_data:/var/lib/redis/data:rw'
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ]
+      interval: 3s
+      retries: 10
+
+  bayanat:
+    container_name: bayanat
+    build:
+      context: .
+      dockerfile: ./flask/Dockerfile.test
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    ports:
+      - '127.0.0.1:5000:5000'
+    volumes:
+      - '${PWD}/reload.ini:/app/reload.ini:rw'
+      - '${PWD}/${ENV_FILE:-.env.test}:/app/.env:ro'
+      - 'bayanat_test_media:/app/enferno/media:rw'
+      - 'bayanat_test_imports:/app/enferno/imports:rw'
+
+volumes:
+  redis_test_data:
+  postgres_test_data:
+  bayanat_test_media:
+  bayanat_test_imports: