docker-compose.yml

services:
  test:
    build:
      context: ./
      dockerfile: Dockerfile
    entrypoint: ["poetry", "run", "pytest","-vv", "-o", "log_cli=true", "--cov-report", "term:skip-covered", "--cov=quotaclimat", "--cov=postgres", "test/"]
    #entrypoint: ["sleep", "12000"] # use to debug the container if needed
    environment:
      ENV: docker
      # CHANNEL: "fr3-idf"
      LOGLEVEL: INFO
      PYTHONPATH: /app
      POSTGRES_USER: user
      POSTGRES_DB: barometre
      POSTGRES_PASSWORD: password
      POSTGRES_HOST: postgres_db
      POSTGRES_PORT: 5432
      MODIN_ENGINE: ray
      COMPARE_DURATION: "true"
    tty: true # colorize terminal
    volumes:
      - ./quotaclimat/:/app/quotaclimat/
      - ./postgres/:/app/postgres/
      - ./test/:/app/test/
      - ./poetry.lock:/app/poetry.lock
      - ./pyproject.toml:/app/pyproject.toml
      - ./alembic:/app/alembic
      - ./alembic.ini:/app/alembic.ini
    depends_on:
      nginxtest:
        condition: service_healthy
      postgres_db:
        condition: service_healthy

  testconsole:
    build:
      context: ./
      dockerfile: Dockerfile
    entrypoint: ["sleep", "12000"] # use to debug the container if needed
    environment:
      ENV: docker
      # CHANNEL: "fr3-idf"
      LOGLEVEL: DEBUG
      PYTHONPATH: /app
      POSTGRES_USER: user
      POSTGRES_DB: barometre
      POSTGRES_PASSWORD: password
      POSTGRES_HOST: postgres_db
      POSTGRES_PORT: 5432
      COMPARE_DURATION: "true"
      MODIN_ENGINE: ray
      MEDIATREE_USER : /run/secrets/username_api
      MEDIATREE_PASSWORD: /run/secrets/pwd_api
      BUCKET: /run/secrets/bucket
      BUCKET_NAME: mediatree
      BUCKET_SECRET: /run/secrets/bucket_secret
      MODIN_CPUS: 4 # "https://modin.readthedocs.io/en/0.11.0/using_modin.html#reducing-or-limiting-the-resources-modin-can-use"
    tty: true # colorize terminal
    secrets:
       - pwd_api
       - username_api
       - bucket
       - bucket_secret
    volumes:
      - ./quotaclimat/:/app/quotaclimat/
      - ./postgres/:/app/postgres/
      - ./test/:/app/test/
      - ./poetry.lock:/app/poetry.lock
      - ./pyproject.toml:/app/pyproject.toml
      - ./alembic:/app/alembic
      - ./alembic.ini:/app/alembic.ini
    depends_on:
      nginxtest:
        condition: service_healthy
      postgres_db:
        condition: service_healthy


  nginxtest: # to test locally webpages
    container_name: nginxtest
    image: nginx:latest
    healthcheck:
      test: ["CMD", "service", "nginx", "status"]
      interval: 5s
      timeout: 5s
      retries: 5
    ports:
      - 8000:80
    volumes:
      - './mockwebsite:/usr/share/nginx/html'

  ingest_to_db:
    container_name: ingest_to_db
    ports:
      - 5000:5000
    build:
      context: ./
      dockerfile: Dockerfile_ingest
    #entrypoint: ["sleep", "1200"] # use to debug the container if needed
    entrypoint: ["python", "quotaclimat/data_ingestion/ingest_db/ingest_sitemap_in_db.py"]
    environment:
      ENV: docker # change me to prod for real cases
      LOGLEVEL: INFO # Change me to info (debug, info, warning, error) to have less log
      PYTHONPATH: /app
      POSTGRES_USER: user
      POSTGRES_DB: barometre
      POSTGRES_PASSWORD: password
      POSTGRES_HOST: postgres_db
      POSTGRES_PORT: 5432
      PORT: 5000
      HEALTHCHECK_SERVER: "0.0.0.0"
    volumes:
      - ./quotaclimat/:/app/quotaclimat/
      - ./postgres/:/app/postgres/
    depends_on:
      nginxtest:
        condition: service_healthy
      postgres_db:
        condition: service_healthy


  stop_word:
    ports:
      - 5002:5002
    build:
      context: ./
      dockerfile: Dockerfile_stop_word
    #entrypoint: ["sleep", "1200"] # use to debug the container if needed
    environment:
      ENV: docker # change me to prod for real cases
      LOGLEVEL: INFO # Change me to info (debug, info, warning, error) to have less log
      PYTHONPATH: /app
      POSTGRES_USER: user
      POSTGRES_DB: barometre
      POSTGRES_PASSWORD: password
      POSTGRES_HOST: postgres_db
      POSTGRES_PORT: 5432
      PORT: 5002
      HEALTHCHECK_SERVER: "0.0.0.0"
      # NUMBER_OF_PREVIOUS_DAYS: 30
      # MIN_REPETITION: 15
      # START_DATE: 1731683697
    volumes:
      - ./quotaclimat/:/app/quotaclimat/
      - ./postgres/:/app/postgres/
      - ./test/:/app/test/
    depends_on:
      nginxtest:
        condition: service_healthy
      postgres_db:
        condition: service_healthy

  postgres_db:
    image: postgres:15
    ports:
      - 5432:5432
    volumes:
      - './pgdata:/var/lib/postgresql/data' # persist data https://stackoverflow.com/a/41650891/3535853
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 5s
      timeout: 5s
      retries: 5
    environment:
      POSTGRES_USER: user
      POSTGRES_DB: barometre
      POSTGRES_PASSWORD: password
    logging: # no logs for postgres container
            driver: none

  mediatree:
    ports:
      - 5050:5050
      - 8265:8265
    build:
      context: ./
      dockerfile: Dockerfile_api_import
    #entrypoint: ["sleep", "1200"] # use to debug the container if needed
    #entrypoint: ["python", "quotaclimat/data_processing/mediatree/api_import.py"]
    environment:
      ENV: docker # change me to prod for real cases
      LOGLEVEL: DEBUG # Change me to info (debug, info, warning, error) to have less log
      PYTHONPATH: /app
      POSTGRES_USER: user
      POSTGRES_DB: barometre
      POSTGRES_PASSWORD: password
      POSTGRES_HOST: postgres_db
      POSTGRES_PORT: 5432
      PORT_HS: 5050 # healthcheck
      HEALTHCHECK_SERVER: "0.0.0.0"
      # SENTRY_DSN: prod_only
      # COMPARE_DURATION: "true"
      UPDATE: "true" # to batch update PG 
      #UPDATE_PROGRAM_ONLY: "true" # to batch update PG but only channel with program
      START_DATE_UPDATE: "2024-02-01" # to batch update PG from a date
      # NUMBER_OF_DAYS: 7
      STOP_WORD_KEYWORD_ONLY: "true"  # speed up execution when we only want to update stop words
      #END_DATE: "2024-02-29" # optional - otherwise end of the month
      BATCH_SIZE: 100 # number of records to update in one batch
      #START_DATE: 1727610071 # to test batch import 
      # NUMBER_OF_PREVIOUS_DAYS: 30 # to query mediatree api only - not used for update
      CHANNEL : fr3-idf # to reimport only one channel
      MEDIATREE_USER : /run/secrets/username_api
      MEDIATREE_PASSWORD:  /run/secrets/pwd_api
      MEDIATREE_AUTH_URL: https://keywords.mediatree.fr/api/auth/token/
      KEYWORDS_URL: https://keywords.mediatree.fr/api/subtitle/ # https://keywords.mediatree.fr/docs/#api-Subtitle-SubtitleList
      MODIN_ENGINE: ray
      MODIN_CPUS: 4 # "https://modin.readthedocs.io/en/0.11.0/using_modin.html#reducing-or-limiting-the-resources-modin-can-use"
      MODIN_MEMORY: 1000000000 # 1Gb
      RAY_memory_usage_threshold: 1
    mem_limit: "1G"
    volumes:
      - ./quotaclimat/:/app/quotaclimat/
      - ./postgres/:/app/postgres/
      - ./test/:/app/test/
    secrets:
       - pwd_api
       - username_api
    depends_on:
      nginxtest:
        condition: service_healthy
      postgres_db:
        condition: service_healthy

  api_to_s3:
    ports:
      - 5666:5666
      - 8265:8265
    build:
      context: ./
      dockerfile: Dockerfile_api_to_s3
    environment:
      ENV: docker # change me to prod for real cases
      LOGLEVEL: DEBUG # Change me to info (debug, info, warning, error) to have less log
      PYTHONPATH: /app
      PORT_HS: 5666 # healthcheck
      HEALTHCHECK_SERVER: "0.0.0.0"
      # SENTRY_DSN: prod_only
      #END_DATE: "2024-02-29" # optional - otherwise end of the month
      # START_DATE: 1727610071 # to test batch import 
      #CHANNEL : fr3-idf # to reimport only one channel
      MEDIATREE_USER : /run/secrets/username_api
      MEDIATREE_PASSWORD: /run/secrets/pwd_api
      BUCKET: /run/secrets/bucket
      BUCKET_SECRET: /run/secrets/bucket_secret
      BUCKET_NAME: mediatree
      MEDIATREE_AUTH_URL: https://keywords.mediatree.fr/api/auth/token/
      KEYWORDS_URL: https://keywords.mediatree.fr/api/subtitle/ # https://keywords.mediatree.fr/docs/#api-Subtitle-SubtitleList
      MODIN_ENGINE: ray
      MODIN_CPUS: 6 # "https://modin.readthedocs.io/en/0.11.0/using_modin.html#reducing-or-limiting-the-resources-modin-can-use"
      MODIN_MEMORY: 12000000000 # 1Gb
      RAY_memory_usage_threshold: 0.95
    volumes:
      - ./quotaclimat/:/app/quotaclimat/
      - ./postgres/:/app/postgres/
      - ./test/:/app/test/
      - ./s3:/app/s3/
    secrets:
       - pwd_api
       - username_api
       - bucket
       - bucket_secret
    depends_on:
      nginxtest:
        condition: service_healthy
      postgres_db:
        condition: service_healthy

  metabase:
      container_name: metabase_barometre
      image: metabase/metabase:latest
      volumes:
        - './mb-data:/metabase-data'  # Volumes where Metabase data will be persisted
      ports:
        - 3000:3000
      environment:
        MB_DB_TYPE: postgres
        MB_DB_DBNAME: barometre
        MB_DB_PORT: 5432
        MB_DB_USER: user
        MB_DB_PASS: password
        MB_DB_HOST: postgres_db
        MB_DB_FILE: "/metabase-data/metabase.db" # https://www.metabase.com/docs/latest/operations-guide/running-metabase-on-docker.html#mounting-a-mapped-file-storage-volume
        JAVA_TIMEZONE: Europe/Paris
      depends_on:
        postgres_db:
          condition: service_healthy

secrets: # https://docs.docker.com/compose/use-secrets/
  pwd_api:
    file: secrets/pwd_api.txt
  username_api:
    file: secrets/username_api.txt
  bucket:
    file: secrets/scw_bucket.txt
  bucket_secret:
    file: secrets/scw_bucket_secret.txt