docker-compose.yml

version: '3' 
services:

  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    volumes:
      - ~/bigdatavault/hadoop/dfs/name:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ~/bdi/config/hadoop-hive.env
    ports:
      - "9870:9870"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.5
  
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    volumes:
      - ~/bigdatavault/hadoop/dfs/data:/hadoop/dfs/data
    env_file:
      - ~/bdi/config/hadoop-hive.env
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    depends_on:
      - namenode
    ports:
      - "9864:9864"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.6

  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:3.1.0
    container_name: hive-metastore-postgresql
    depends_on:
      - datanode
    networks:
      net_lucas:
        ipv4_address: 172.27.1.9
    
  
  hive-metastore:
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-metastore
    env_file:
      - ~/bdi/config/hadoop-hive.env
    command: /opt/hive/bin/hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
    ports:
      - "9083:9083"
    depends_on:
      - hive-metastore-postgresql
    networks:
      net_lucas:
        ipv4_address: 172.27.1.8
  
  hive-server:
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-server
    env_file:
      - ~/bdi/config/hadoop-hive.env
    environment:
      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
      SERVICE_PRECONDITION: "hive-metastore:9083"
    ports:
      - "10000:10000"
    depends_on:
      - hive-metastore
    networks:
      net_lucas:
        ipv4_address: 172.27.1.7
  
  
  spark-master:
    image: bde2020/spark-master:3.0.1-hadoop3.2
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    env_file:
      - ~/bdi/config/hadoop-hive.env
    depends_on:
       - datanode
    networks:
      net_lucas:
        ipv4_address: 172.27.1.10
     
  spark-worker-1:
    image: bde2020/spark-worker:3.0.1-hadoop3.2
    container_name: spark-worker-1
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
    depends_on:
      - spark-master
    ports:
      - 8081:8081
    env_file:
      - ~/bdi/config/hadoop-hive.env
    networks:
      net_lucas:
        ipv4_address: 172.27.1.11

  spark-worker-2:
    image: bde2020/spark-worker:3.0.1-hadoop3.2
    container_name: spark-worker-2
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
    depends_on:
      - spark-worker-1
    ports:
      - 8082:8082
    env_file:
      - ~/bdi/config/hadoop-hive.env
    networks:
      net_lucas:
        ipv4_address: 172.27.1.12

  zeppelin:
    image: lucas/zeppelin
    container_name: zeppelin
    environment:
      ZEPPELIN_NOTEBOOK_DIR: "/notebook"
      ZEPPELIN_LOG_DIR: "/logs"
      CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
      SPARK_MASTER: "spark://spark-master:7077"
      MASTER: "spark://spark-master:7077"
      SPARK_MASTER_URL: spark://spark-master:7077
      ZEPPELIN_PORT: 8080
      ZEPPELIN_JAVA_OPTS:
        -Dspark.driver.memory=1g
        -Dspark.executor.memory=2g
    ports:
      - 19090:8080
    env_file:
      - ~/bdi/config/hadoop-hive.env
    volumes:
      - ~/bigdatavault/zeppelin/logs:/logs:rw
      - ~/bigdatavault/zeppelin/data:/data:rw
      - ~/bigdatavault/zeppelin/notebook:/notebook:rw
    networks:
      net_lucas:
        ipv4_address: 172.27.1.13

  mysql:
      image: mysql:5.7
      container_name: db_mysql
      ports:
          - "33061:3306"
      command: --init-file /data/application/init.sql
      volumes:
          - ~/bigdatavault/mysql/data:/var/lib/mysql
          - ~/bdi/config/init.sql:/data/application/init.sql
          - ~/bdi/etc/mysql/mysql.conf.d/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
      environment:
          MYSQL_ROOT_USER: root
          MYSQL_ROOT_PASSWORD: secret
          MYSQL_DATABASE: hue
          MYSQL_USER: root
          MYSQL_PASSWORD: secret
      networks:
        net_lucas:
          ipv4_address: 172.27.1.14

  hue:
      image: gethue/hue:latest
      hostname: hue
      container_name: hue
      dns: 8.8.8.8
      ports:
      - "8888:8888"
      volumes:
        - ~/bdi/config/hue-overrides.ini:/usr/share/hue/desktop/conf/z-hue.ini
      depends_on:
      - "mysql"
      - "hive-server"
      networks:
        net_lucas:
          ipv4_address: 172.27.1.15

  zookeeper:
    image: zookeeper:3.6
    container_name: zookeeper
    ports:
      - "2181:2181"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.16

  kafka:
    image: wurstmeister/kafka:2.12-2.5.0
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 172.27.1.32
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_BROKER_ID: 2
    depends_on:
      - "kafka-4"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.32

  kafka-3:
    image: wurstmeister/kafka:2.12-2.5.0
    container_name: kafka-3
    ports:
      - "9093:9093"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 172.27.1.33
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_BROKER_ID: 3
    depends_on:
      - "zookeeper"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.33

  kafka-4:
    image: wurstmeister/kafka:2.12-2.5.0
    container_name: kafka-4
    ports:
      - "9094:9094"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 172.27.1.34
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_BROKER_ID: 4
    depends_on:
      - "kafka-3"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.34

  streamsets:
    image: streamsets/datacollector:3.13.0-latest
    container_name: streamsets
    ports:
      - "18630:18630"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.18  

  mongodb:
    image: mongo
    container_name: mongo1
    volumes:
      - ~/bigdatavault/mdata:/data/db
    ports:
      - 27017:27017
    command: --auth
    networks:
      net_lucas:
        ipv4_address: 172.27.1.19

  redis:
    image: redis
    container_name: redis1
    volumes:
      - ~/bigdatavault/rdata:/data
      - ~/bdi/config/redis.conf:/etc/redis/redis.conf
    ports:
      - 16379:6379
    command: redis-server /etc/redis/redis.conf --appendonly yes
    networks:
      net_lucas:
        ipv4_address: 172.27.1.20

  kylin:
    image: apachekylin/apache-kylin-standalone:3.1.0
    container_name: kylin
    ports:
      - "7070:7070"
      - "8088:8088"
      - "50070:50070"
      - "8032:8032"
      - "8042:8042"
      - "16010:16010"
    networks:
      net_lucas:
        ipv4_address: 172.27.1.21


  presto:
    image: prestosql/presto
    container_name: presto
    ports:
      - "18080:8080"
    volumes:
      - ~/bdi/etc/hive.properties:/usr/lib/presto/default/etc/catalog/hive.properties
      - ~/bdi/etc/kudu.properties:/usr/lib/presto/default/etc/catalog/kudu.properties
      - ~/bdi/etc/hadoop/core-site.xml:/etc/hadoop/core-site.xml
      - ~/bdi/etc/hadoop/hdfs-site.xml:/etc/hadoop/hdfs-site.xml
    networks:
      net_lucas:
        ipv4_address: 172.27.1.22

  nifi:
    image: apache/nifi:latest
    container_name: nifi
    ports: 
      - "29090:9090"
    environment:
      - NIFI_WEB_HTTP_PORT=9090
    volumes:
      - ~/bdi/resources/:/bdi/resources/
    networks:
      - net_lucas

  kudu-master-1:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-master-1
    ports:
      - "7051:7051"
      - "8051:8051"
    networks: 
      - net_lucas
    command: ["master"]
    volumes:
      - kudu-master-1:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      # TODO: Use `host.docker.internal` instead of KUDU_QUICKSTART_IP when it
      # works on Linux (https://github.com/docker/for-linux/issues/264)
      - >
        MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master
        --rpc_bind_addresses=0.0.0.0:7051
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP:?Please set KUDU_QUICKSTART_IP environment variable}:7051
        --webserver_port=8051
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8051
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-master-2:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-master-2
    ports:
      - "7151:7151"
      - "8151:8151"
    networks: 
      - net_lucas
    command: ["master"]
    volumes:
      - kudu-master-2:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master
        --rpc_bind_addresses=0.0.0.0:7151
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7151
        --webserver_port=8151
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8151
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-master-3:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-master-3
    ports:
      - "7251:7251"
      - "8251:8251"
    networks: 
      - net_lucas
    command: ["master"]
    volumes:
      - kudu-master-3:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master
        --rpc_bind_addresses=0.0.0.0:7251
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7251
        --webserver_port=8251
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8251
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-tserver-1:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-tserver-1
    depends_on:
      - kudu-master-1
      - kudu-master-2
      - kudu-master-3
    ports:
      - "7050:7050"
      - "8050:8050"
    networks: 
      - net_lucas
    command: ["tserver"]
    volumes:
      - kudu-tserver-1:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
        --rpc_bind_addresses=0.0.0.0:7050
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7050
        --webserver_port=8050
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8050
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-tserver-2:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-tserver-2
    depends_on:
      - kudu-master-1
      - kudu-master-2
      - kudu-master-3
    ports:
      - "7150:7150"
      - "8150:8150"
    networks: 
      - net_lucas
    command: ["tserver"]
    volumes:
      - kudu-tserver-2:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
        --rpc_bind_addresses=0.0.0.0:7150
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7150
        --webserver_port=8150
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8150
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-tserver-3:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-tserver-3
    depends_on:
      - kudu-master-1
      - kudu-master-2
      - kudu-master-3
    ports:
      - "7250:7250"
      - "8250:8250"
    networks: 
      - net_lucas
    command: ["tserver"]
    volumes:
      - kudu-tserver-3:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
        --rpc_bind_addresses=0.0.0.0:7250
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7250
        --webserver_port=8250
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8250
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-tserver-4:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-tserver-4
    depends_on:
      - kudu-master-1
      - kudu-master-2
      - kudu-master-3
    ports:
      - "7350:7350"
      - "8350:8350"
    networks: 
      - net_lucas
    command: ["tserver"]
    volumes:
      - kudu-tserver-4:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
        --rpc_bind_addresses=0.0.0.0:7350
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7350
        --webserver_port=8350
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8350
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  kudu-tserver-5:
    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
    container_name: kudu-tserver-5
    depends_on:
      - kudu-master-1
      - kudu-master-2
      - kudu-master-3
      - kudu-tserver-1
      - kudu-tserver-2
      - kudu-tserver-3
      - kudu-tserver-4
    ports:
      - "7450:7450"
      - "8450:8450"
    networks: 
      - net_lucas
    command: ["tserver"]
    volumes:
      - kudu-tserver-5:/var/lib/kudu
    environment:
      - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251
      - >
        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
        --rpc_bind_addresses=0.0.0.0:7450
        --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7450
        --webserver_port=8450
        --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8450
        --webserver_doc_root=/opt/kudu/www
        --stderrthreshold=0
        --use_hybrid_clock=false
  impala:
    image: apache/kudu:impala-latest
    container_name: impala
    ports: 
      - "21000:21000"
      - "21050:21050"
      - "25000:25000"
      - "25010:25010"
      - "25020:25020"
    volumes:
      - ~/bdi/resources/:/bdi/resources/
    networks:
      - net_lucas


volumes:
  kudu-master-1:
  kudu-master-2:
  kudu-master-3:
  kudu-tserver-1:
  kudu-tserver-2:
  kudu-tserver-3:
  kudu-tserver-4:
  kudu-tserver-5:


networks:
  net_lucas:
    ipam:
      driver: default
      config:
        - subnet: 172.27.0.0/16