fix(airflow): merge scheduler and api-server into single container
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 33s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m6s
Build and Push Docker Images / Build Integrator (push) Successful in 57s
Build and Push Docker Images / Build Kestra Init (push) Successful in 32s
Build and Push Docker Images / Build Pipeline (Meltano + dbt + Airflow) (push) Successful in 31s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 0s
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 33s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m6s
Build and Push Docker Images / Build Integrator (push) Successful in 57s
Build and Push Docker Images / Build Kestra Init (push) Successful in 32s
Build and Push Docker Images / Build Pipeline (Meltano + dbt + Airflow) (push) Successful in 31s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 0s
With LocalExecutor, tasks run in the scheduler process and logs are written locally. Running api-server and scheduler in separate containers meant the api-server couldn't read task logs (empty hostname in log fetch URL). Combining them into one container eliminates the issue — logs are always on the local filesystem. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -184,11 +184,11 @@ services:
|
|||||||
retries: 3
|
retries: 3
|
||||||
start_period: 15s
|
start_period: 15s
|
||||||
|
|
||||||
# ── Airflow API Server + UI (at :8080) ────────────────────────────────
|
# ── Airflow (api-server + scheduler in one container) ─────────────────
|
||||||
airflow-api-server:
|
airflow:
|
||||||
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
|
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
|
||||||
container_name: schoolcompare_airflow_api
|
container_name: schoolcompare_airflow
|
||||||
command: bash -c "rm -f /opt/airflow/airflow.cfg && exec airflow api-server --port 8080"
|
command: bash -c "rm -f /opt/airflow/airflow.cfg && airflow scheduler & exec airflow api-server --port 8080"
|
||||||
ports:
|
ports:
|
||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
environment:
|
environment:
|
||||||
@@ -198,8 +198,6 @@ services:
|
|||||||
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
|
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
|
||||||
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
|
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
|
||||||
AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
|
AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
|
||||||
AIRFLOW__LOGGING__WORKER_LOG_SERVER_HOST: airflow-scheduler
|
|
||||||
AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT: "8793"
|
|
||||||
PG_HOST: sc_database
|
PG_HOST: sc_database
|
||||||
PG_PORT: "5432"
|
PG_PORT: "5432"
|
||||||
PG_USER: ${DB_USERNAME}
|
PG_USER: ${DB_USERNAME}
|
||||||
@@ -207,8 +205,6 @@ services:
|
|||||||
PG_DATABASE: ${DB_DATABASE_NAME}
|
PG_DATABASE: ${DB_DATABASE_NAME}
|
||||||
TYPESENSE_URL: http://typesense:8108
|
TYPESENSE_URL: http://typesense:8108
|
||||||
TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
|
TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
|
||||||
volumes:
|
|
||||||
- airflow_logs:/opt/airflow/logs
|
|
||||||
depends_on:
|
depends_on:
|
||||||
sc_database:
|
sc_database:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -222,37 +218,6 @@ services:
|
|||||||
retries: 5
|
retries: 5
|
||||||
start_period: 60s
|
start_period: 60s
|
||||||
|
|
||||||
# ── Airflow Scheduler ────────────────────────────────────────────────
|
|
||||||
airflow-scheduler:
|
|
||||||
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
|
|
||||||
container_name: schoolcompare_airflow_scheduler
|
|
||||||
command: bash -c "rm -f /opt/airflow/airflow.cfg && exec airflow scheduler"
|
|
||||||
environment:
|
|
||||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
|
||||||
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
|
|
||||||
AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
|
|
||||||
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
|
|
||||||
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
|
|
||||||
AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
|
|
||||||
AIRFLOW__LOGGING__WORKER_LOG_SERVER_HOST: airflow-scheduler
|
|
||||||
AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT: "8793"
|
|
||||||
PG_HOST: sc_database
|
|
||||||
PG_PORT: "5432"
|
|
||||||
PG_USER: ${DB_USERNAME}
|
|
||||||
PG_PASSWORD: ${DB_PASSWORD}
|
|
||||||
PG_DATABASE: ${DB_DATABASE_NAME}
|
|
||||||
TYPESENSE_URL: http://typesense:8108
|
|
||||||
TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
|
|
||||||
volumes:
|
|
||||||
- airflow_logs:/opt/airflow/logs
|
|
||||||
depends_on:
|
|
||||||
sc_database:
|
|
||||||
condition: service_healthy
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
restart: unless-stopped
|
|
||||||
hostname: airflow-scheduler
|
|
||||||
|
|
||||||
# ── Airflow DB Init (one-shot) ───────────────────────────────────────
|
# ── Airflow DB Init (one-shot) ───────────────────────────────────────
|
||||||
airflow-init:
|
airflow-init:
|
||||||
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
|
image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
|
||||||
@@ -282,4 +247,3 @@ volumes:
|
|||||||
kestra_storage:
|
kestra_storage:
|
||||||
supplementary_data:
|
supplementary_data:
|
||||||
typesense_data:
|
typesense_data:
|
||||||
airflow_logs:
|
|
||||||
|
|||||||
Reference in New Issue
Block a user