docker-compose.yml

version: "3"

services:
  flowman:
    image: dimajix/flowman:1.1.0-SNAPSHOT-oss-spark3.4-hadoop3.3
    command: bash
    depends_on:
      - mariadb
      - pushgateway
      - flowman-server
    volumes:
      - ./conf:/opt/flowman/conf
      - ./flow:/home/flowman/flow
      - ./demo:/home/flowman/demo
    environment:
      - http_proxy=${http_proxy}
      - https_proxy=${https_proxy}
      - SPARK_MASTER=local[10]
      - JDBC_DRIVER=org.mariadb.jdbc.Driver
      - JDBC_URL=jdbc:mariadb://mariadb/weather
      - JDBC_USERNAME=weather
      - JDBC_PASSWORD=weather
      - HISTORY_JDBC_DRIVER=org.mariadb.jdbc.Driver
      - HISTORY_JDBC_URL=jdbc:mariadb://mariadb/flowman_history
      - HISTORY_JDBC_USERNAME=flowman_history
      - HISTORY_JDBC_PASSWORD=flowman_history
      - URL_PROMETHEUS_PUSHGW=http://pushgateway:9091

  # The Flowman History Server
  flowman-server:
    image: dimajix/flowman:1.1.0-SNAPSHOT-oss-spark3.4-hadoop3.3
    command: flowman-server
    depends_on:
      - mariadb
    volumes:
      - ./conf:/opt/flowman/conf
    environment:
      - http_proxy=${http_proxy}
      - https_proxy=${https_proxy}
      - SPARK_MASTER=local[1]
      - HISTORY_JDBC_DRIVER=org.mariadb.jdbc.Driver
      - HISTORY_JDBC_URL=jdbc:mariadb://mariadb/flowman_history
      - HISTORY_JDBC_USERNAME=flowman_history
      - HISTORY_JDBC_PASSWORD=flowman_history
    ports:
      - 8080:8080

  # The MariaDB server will host three databases
  #  - The "weather" database, which contains the transformed result data
  #  - The "flowman_history" database, which is used by the Flowman History Server
  #  - The "flowman_metrics" database, which contains additional execution metrics
  mariadb:
    image: mariadb:10.9.3
    environment:
      - MARIADB_ROOT_PASSWORD=rootpwd
    command:
      - --sql-mode=STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,ANSI_QUOTES
    volumes:
      - ./docker/mariadb/init.sql:/docker-entrypoint-initdb.d/init.sql:z
    ports:
      - 3306:3306

  # Prometheus server will store Flowman execution metrics to be displayed in Grafana
  prometheus:
    image: prom/prometheus:v2.37.1
    volumes:
      - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:z
    ports:
      - 9090:9090

  # The Prometheus Push Gateway is required by Flowman for submitting execution metrics
  pushgateway:
    image: prom/pushgateway:v1.4.3
    depends_on:
      - prometheus
    ports:
      - 9091:9091

  grafana:
    image: grafana/grafana:9.1.6
    depends_on:
      - prometheus
    ports:
      - 3000:3000