Files
school_compare/docker-compose.yml
Tudor 8f02b5125e
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 35s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m9s
Build and Push Docker Images / Build Integrator (push) Successful in 56s
Build and Push Docker Images / Build Kestra Init (push) Successful in 32s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 1s
feat(pipeline): add Meltano + dbt + Airflow ELT pipeline scaffold
Replaces the hand-rolled integrator with a production-grade ELT pipeline
using Meltano (Singer taps), dbt Core (medallion architecture), and
Apache Airflow (orchestration). Adds Typesense for search and PostGIS
for geospatial queries.

- 6 custom Singer taps (GIAS, EES, Ofsted, Parent View, FBIT, IDACI)
- dbt project: 12 staging, 5 intermediate, 12 mart models
- 3 Airflow DAGs (daily/monthly/annual schedules)
- Typesense sync + batch geocoding scripts
- docker-compose: add Airflow, Typesense; upgrade to PostGIS
- Portainer stack definition matching live deployment topology

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-26 08:37:53 +00:00

172 lines
4.8 KiB
YAML

version: '3.8'
services:
# PostgreSQL Database with PostGIS
db:
image: postgis/postgis:16-3.4-alpine
container_name: schoolcompare_db
environment:
POSTGRES_USER: schoolcompare
POSTGRES_PASSWORD: schoolcompare
POSTGRES_DB: schoolcompare
volumes:
- postgres_data:/var/lib/postgresql/data
ports:
- "5432:5432"
networks:
- schoolcompare-network
restart: unless-stopped
healthcheck:
test: ["CMD-SHELL", "pg_isready -U schoolcompare"]
interval: 10s
timeout: 5s
retries: 5
start_period: 10s
# FastAPI Backend
backend:
image: privaterepo.sitaru.org/tudor/school_compare-backend:latest
container_name: schoolcompare_backend
ports:
- "8000:80"
environment:
DATABASE_URL: postgresql://schoolcompare:schoolcompare@db:5432/schoolcompare
PYTHONUNBUFFERED: 1
ADMIN_API_KEY: ${ADMIN_API_KEY:-changeme}
TYPESENSE_URL: http://typesense:8108
TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
volumes:
- ./data:/app/data:ro
depends_on:
db:
condition: service_healthy
networks:
- schoolcompare-network
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:80/api/data-info"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
# Next.js Frontend
nextjs:
image: privaterepo.sitaru.org/tudor/school_compare-frontend:latest
container_name: schoolcompare_nextjs
ports:
- "3000:3000"
environment:
NODE_ENV: production
NEXT_PUBLIC_API_URL: http://localhost:8000/api
FASTAPI_URL: http://backend:80/api
TYPESENSE_URL: http://typesense:8108
TYPESENSE_API_KEY: ${TYPESENSE_SEARCH_KEY:-changeme}
depends_on:
backend:
condition: service_healthy
networks:
- schoolcompare-network
restart: unless-stopped
healthcheck:
test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
# Typesense — search engine
typesense:
image: typesense/typesense:27.1
container_name: schoolcompare_typesense
ports:
- "8108:8108"
environment:
TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
TYPESENSE_DATA_DIR: /data
volumes:
- typesense_data:/data
networks:
- schoolcompare-network
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:8108/health"]
interval: 15s
timeout: 5s
retries: 5
start_period: 10s
# Apache Airflow — workflow orchestrator (UI at http://localhost:8080)
airflow-webserver:
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
container_name: schoolcompare_airflow_webserver
command: airflow webserver --port 8080
ports:
- "8080:8080"
environment: &airflow-env
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://schoolcompare:schoolcompare@db:5432/schoolcompare
AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "false"
PG_HOST: db
PG_PORT: "5432"
PG_USER: schoolcompare
PG_PASSWORD: schoolcompare
PG_DATABASE: schoolcompare
TYPESENSE_URL: http://typesense:8108
TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
volumes:
- ./pipeline/dags:/opt/pipeline/dags:ro
depends_on:
db:
condition: service_healthy
networks:
- schoolcompare-network
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
airflow-scheduler:
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
container_name: schoolcompare_airflow_scheduler
command: airflow scheduler
environment: *airflow-env
volumes:
- ./pipeline/dags:/opt/pipeline/dags:ro
depends_on:
db:
condition: service_healthy
networks:
- schoolcompare-network
restart: unless-stopped
# One-shot: initialise Airflow metadata DB
airflow-init:
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
container_name: schoolcompare_airflow_init
command: >
bash -c "
airflow db migrate &&
airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@localhost || true
"
environment: *airflow-env
depends_on:
db:
condition: service_healthy
networks:
- schoolcompare-network
restart: "no"
networks:
schoolcompare-network:
driver: bridge
volumes:
postgres_data:
typesense_data: