docker-compose.yml

version: "3"
services:

    # scheduler & monitoring service
    airflow-webserver:
        hostname: myairflow
        container_name: airflow_container
        image: 'puckel/docker-airflow:latest'
        ports:
            - '8085:8080'
        networks:
            - dataworld
        volumes:
            - airflow-data:/usr/local/airflow/data
            - ./airflow/logs:/usr/local/airflow/logs
            - ./airflow/dags:/usr/local/airflow/dags
            - ./airflow/requirements/requirements.txt:/requirements.txt
            - ./airflow/airflow.cfg:/usr/local/airflow/airflow.cfg
        environment:
            - GUNICORN_CMD_ARGS=--log-level warning
        restart: on-failure
        healthcheck:
            test: ["CMD", "curl", "-f", "http://myairflow:8080/admin/"]
            interval: 30s
            timeout: 20s
            retries: 3

    # configuration manager for NiFi
    zookeeper:
        hostname: myzookeeper
        container_name: zookeeper_container
        image: 'bitnami/zookeeper:latest'
        environment:
            - ALLOW_ANONYMOUS_LOGIN=yes
        networks:
            - dataworld
        restart: always

    # data extraction, transformation and load service
    nifi:
        hostname: mynifi
        container_name: nifi_container
        image: 'apache/nifi:latest'
        ports:
            - '8091:8080'
        networks:
            - dataworld
        volumes:
            - ./nifi/jdbc:/opt/nifi/nifi-current/jdbc
            - ./nifi/credentials:/opt/nifi/nifi-current/credentials
            - ./nifi/logback.xml:/opt/nifi/nifi-current/conf/logback.xml
        environment:
            - NIFI_WEB_HTTP_PORT=8080
            - NIFI_CLUSTER_IS_NODE=true
            - NIFI_CLUSTER_NODE_PROTOCOL_PORT=8082
            - NIFI_ZK_CONNECT_STRING=myzookeeper:2181
            - NIFI_ELECTION_MAX_WAIT=30 sec
            - NIFI_SENSITIVE_PROPS_KEY='12345678901234567890A'
            - NIFI_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL=60 sec
        restart: on-failure
        healthcheck:
            test: ["CMD", "curl", "-f", "http://mynifi:8080/nifi/"]
            interval: 30s
            timeout: 20s
            retries: 3

    # version control for nifi flows
    registry:
        hostname: myregistry
        container_name: registry_container
        image: 'apache/nifi-registry:latest'
        restart: on-failure
        ports:
            - "18080:18080"
        environment:
            - LOG_LEVEL=ERROR
            - NIFI_REGISTRY_DB_DIR=/opt/nifi-registry/nifi-registry-current/database
            - NIFI_REGISTRY_FLOW_PROVIDER=file
            - NIFI_REGISTRY_FLOW_STORAGE_DIR=/opt/nifi-registry/nifi-registry-current/flow_storage
        volumes:
            - ./nifi_registry/database:/opt/nifi-registry/nifi-registry-current/database
            - ./nifi_registry/flow_storage:/opt/nifi-registry/nifi-registry-current/flow_storage
        networks:
            - dataworld
        healthcheck:
            test: ["CMD", "curl", "-f", "http://myregistry:18080/nifi-registry/"]
            interval: 30s
            timeout: 20s
            retries: 3

    # relational database
    postgres:
        hostname: mypostgres
        container_name: postgres_container
        image: 'postgres:14-bullseye'  # latest image as of 2021-11-08
        environment:
            POSTGRES_USER: 'postgres'
            POSTGRES_PASSWORD: 'postgres'
            PGDATA: /data/postgres
        volumes:
            - postgres:/data/postgres
        ports:
            - "5432:5432"
        networks:
            - dataworld
        restart: on-failure
        
    # database administration tool
    pgadmin:
        hostname: mypgadmin
        container_name: pgadmin_container
        image: 'dpage/pgadmin4:6.1'  # latest image as of 2021-11-08
        environment:
            PGADMIN_DEFAULT_EMAIL: 'pgadmin4@pgadmin.org'
            PGADMIN_DEFAULT_PASSWORD: 'admin'
            PGADMIN_CONFIG_SERVER_MODE: 'False'
        volumes:
            - pgadmin:/var/lib/pgadmin
            - ./pgadmin:/home
        ports:
            - "5050:80"
        networks:
            - dataworld
        restart: on-failure
        
    # object storage
    minio:
        hostname: myminio
        container_name: minio_container
        image: 'bitnami/minio:latest'
        environment:
            MINIO_ROOT_USER: minio_admin
            MINIO_ROOT_PASSWORD: minio_password
        ports:
            - '9000:9000'
            - '9001:9001'
        volumes:
            - './minio/data:/data'
        networks:
            - dataworld
        healthcheck:
            test: ["CMD", "curl", "-f", "http://myminio:9000/minio/health/live"]
            interval: 30s
            timeout: 20s
            retries: 3

    portainer:
        image: portainer/portainer-ce:latest
        container_name: portainer
        restart: unless-stopped
        security_opt:
            - no-new-privileges:true
        volumes:
            - /etc/localtime:/etc/localtime:ro
            - /var/run/docker.sock:/var/run/docker.sock:ro
            - portainer-data:/data
        ports:
            - 9009:9000

volumes:
    airflow-data:
    postgres:
    pgadmin:
    portainer-data:

networks:
    dataworld:
        driver: bridge