school_compare/docker-compose.portainer.yml

# Portainer Stack Definition for School Compare
#
# Portainer environment variables (set in Portainer UI -> Stack -> Environment):
#   DB_USERNAME            — PostgreSQL username
#   DB_PASSWORD            — PostgreSQL password
#   DB_DATABASE_NAME       — PostgreSQL database name
#   ADMIN_API_KEY          — Backend admin API key
#   TYPESENSE_API_KEY      — Typesense admin API key
#   TYPESENSE_SEARCH_KEY   — Typesense search-only key (exposed to frontend)
#   UNLEASH_URL            — http://<unleash-ip>:4242/api (empty = all flags off)
#   UNLEASH_API_TOKEN      — Unleash *client* token, environment: production
#   PAYLOAD_SECRET         — Payload CMS encryption secret. REQUIRED: long and
#                            random. Changing it invalidates every admin
#                            session. Staging MUST use a different value.
#   AIRFLOW_ADMIN_USER     — Airflow admin username (default: admin)
#   AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server
#                            refuses to start without it, rather than falling
#                            back to a generated one that changes on restart.

services:

  # ── PostgreSQL ────────────────────────────────────────────────────────
  sc_database:
    container_name: sc_postgres
    image: postgis/postgis:18-3.6-alpine
    environment:
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_USER: ${DB_USERNAME}
      POSTGRES_DB: ${DB_DATABASE_NAME}
    volumes:
      - postgres_data:/var/lib/postgresql
    shm_size: 128mb
    networks:
      backend: {}
      macvlan:
        ipv4_address: 10.0.1.189
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    restart: unless-stopped

  # ── FastAPI Backend ───────────────────────────────────────────────────
  backend:
    image: privaterepo.sitaru.org/tudor/school_compare-backend:prod
    container_name: schoolcompare_backend
    environment:
      DATABASE_URL: postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
      PYTHONUNBUFFERED: 1
      ADMIN_API_KEY: ${ADMIN_API_KEY:-changeme}
      TYPESENSE_URL: http://typesense:8108
      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
      # Unset means every feature flag is False — the correct dark state for an
      # environment with no Unleash, not a failure.
      UNLEASH_URL: ${UNLEASH_URL:-}
      UNLEASH_API_TOKEN: ${UNLEASH_API_TOKEN:-}
    volumes:
      - unleash_cache:/app/.unleash
    depends_on:
      sc_database:
        condition: service_healthy
    networks:
      - backend
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:80/api/data-info"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s

  # ── Next.js Frontend ──────────────────────────────────────────────────
  frontend:
    image: privaterepo.sitaru.org/tudor/school_compare-frontend:prod
    container_name: schoolcompare_nextjs
    environment:
      - NODE_ENV=production
      - NEXT_PUBLIC_API_URL=http://localhost:8000/api
      - FASTAPI_URL=http://backend:80/api
      - TYPESENSE_URL=http://typesense:8108
      - TYPESENSE_API_KEY=${TYPESENSE_SEARCH_KEY:-changeme}
      # Payload CMS runs inside this container. It reaches Postgres over the
      # `backend` network and keeps its tables in the `payload` schema, so no
      # pipeline operation on `public` can touch blog content.
      - DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
      # Same :? form as AIRFLOW_ADMIN_PASSWORD: refuse to start rather than
      # boot with an empty secret and silently accept forged sessions.
      - PAYLOAD_SECRET=${PAYLOAD_SECRET:?set PAYLOAD_SECRET in the Portainer stack environment}
    volumes:
      # Blog images. Not reproducible from the pipeline — must be backed up.
      - payload_media:/app/media
    depends_on:
      backend:
        condition: service_healthy
      sc_database:
        condition: service_healthy
    networks:
      backend: {}
      macvlan:
        ipv4_address: 10.0.1.150
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # ── Typesense Search Engine ───────────────────────────────────────────
  typesense:
    image: typesense/typesense:30.1
    container_name: schoolcompare_typesense
    environment:
      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
      TYPESENSE_DATA_DIR: /data
    volumes:
      - typesense_data:/data
    networks:
      - backend
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "cat < /dev/tcp/localhost/8108"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 10s

  # ── Airflow API Server + UI ───────────────────────────────────────────
  airflow-api-server:
    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
    container_name: schoolcompare_airflow_api
    # The simple auth manager generates a random password on first start and
    # writes it to a file, so every container restart invalidates the last one.
    # Writing the file ourselves from an environment variable makes the login
    # deterministic. Airflow does not generate anything when the file exists.
    #
    # Built with python rather than echo/printf so a password containing quotes,
    # backslashes or spaces is escaped correctly by json.dumps. An unset
    # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling
    # back to a generated password would silently undo the point of this.
    command:
      - bash
      - -c
      - |
        set -euo pipefail
        mkdir -p /opt/airflow
        python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))"
        exec airflow api-server --port 8080
    ports:
      - "8080:8080"
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-api-server:8080/execution/
      AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512"
      AIRFLOW__API_AUTH__JWT_ISSUER: airflow
      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json
      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?set AIRFLOW_ADMIN_PASSWORD in the Portainer stack environment}
      AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
      PG_HOST: sc_database
      PG_PORT: "5432"
      PG_USER: ${DB_USERNAME}
      PG_PASSWORD: ${DB_PASSWORD}
      PG_DATABASE: ${DB_DATABASE_NAME}
      TYPESENSE_URL: http://typesense:8108
      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
    volumes:
      - airflow_logs:/opt/airflow/logs
    depends_on:
      sc_database:
        condition: service_healthy
    networks:
      - backend
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/api/v2/monitor/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # ── Airflow Scheduler ──────────────────────────────────────────────
  airflow-scheduler:
    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
    container_name: schoolcompare_airflow_scheduler
    command: airflow scheduler
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-api-server:8080/execution/
      AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512"
      AIRFLOW__API_AUTH__JWT_ISSUER: airflow
      AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
      PG_HOST: sc_database
      PG_PORT: "5432"
      PG_USER: ${DB_USERNAME}
      PG_PASSWORD: ${DB_PASSWORD}
      PG_DATABASE: ${DB_DATABASE_NAME}
      TYPESENSE_URL: http://typesense:8108
      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
    volumes:
      - airflow_logs:/opt/airflow/logs
    depends_on:
      sc_database:
        condition: service_healthy
    networks:
      - backend
    restart: unless-stopped

  # ── Airflow DB Init (one-shot) ───────────────────────────────────────
  airflow-init:
    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
    container_name: schoolcompare_airflow_init
    command: bash -c "airflow db migrate && airflow dags delete school_data_daily -y 2>/dev/null; airflow dags delete school_data_monthly_ofsted -y 2>/dev/null; airflow dags delete school_data_annual_ees -y 2>/dev/null; airflow dags reserialize"
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-api-server:8080/execution/
      AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512"
      AIRFLOW__API_AUTH__JWT_ISSUER: airflow
    depends_on:
      sc_database:
        condition: service_healthy
    networks:
      - backend
    restart: "no"

networks:
  backend:
    driver: bridge
  macvlan:
    external:
      name: macvlan

volumes:
  postgres_data:
  typesense_data:
  airflow_logs:
  unleash_cache:
  payload_media: