From 07869738c0638c7c524eb696418069c4919d72fb Mon Sep 17 00:00:00 2001
From: Tudor <tudor@sitaru.org>
Date: Thu, 26 Mar 2026 12:16:18 +0000
Subject: [PATCH] fix(airflow): merge scheduler and api-server into single
 container
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With LocalExecutor, tasks run in the scheduler process and logs are
written locally. Running api-server and scheduler in separate containers
meant the api-server couldn't read task logs (empty hostname in log
fetch URL). Combining them into one container eliminates the issue —
logs are always on the local filesystem.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 docker-compose.portainer.yml | 44 ++++--------------------------------
 1 file changed, 4 insertions(+), 40 deletions(-)

diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml
index c54a333..6287969 100644
--- a/docker-compose.portainer.yml
+++ b/docker-compose.portainer.yml
@@ -184,11 +184,11 @@ services:
       retries: 3
       start_period: 15s
 
-  # ── Airflow API Server + UI (at :8080) ────────────────────────────────
-  airflow-api-server:
+  # ── Airflow (api-server + scheduler in one container) ─────────────────
+  airflow:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
-    container_name: schoolcompare_airflow_api
-    command: bash -c "rm -f /opt/airflow/airflow.cfg && exec airflow api-server --port 8080"
+    container_name: schoolcompare_airflow
+    command: bash -c "rm -f /opt/airflow/airflow.cfg && airflow scheduler & exec airflow api-server --port 8080"
     ports:
       - "8080:8080"
     environment:
@@ -198,8 +198,6 @@ services:
       AIRFLOW__CORE__LOAD_EXAMPLES: "false"
       AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
       AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
-      AIRFLOW__LOGGING__WORKER_LOG_SERVER_HOST: airflow-scheduler
-      AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT: "8793"
       PG_HOST: sc_database
       PG_PORT: "5432"
       PG_USER: ${DB_USERNAME}
@@ -207,8 +205,6 @@ services:
       PG_DATABASE: ${DB_DATABASE_NAME}
       TYPESENSE_URL: http://typesense:8108
       TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
-    volumes:
-      - airflow_logs:/opt/airflow/logs
     depends_on:
       sc_database:
         condition: service_healthy
@@ -222,37 +218,6 @@ services:
       retries: 5
       start_period: 60s
 
-  # ── Airflow Scheduler ────────────────────────────────────────────────
-  airflow-scheduler:
-    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
-    container_name: schoolcompare_airflow_scheduler
-    command: bash -c "rm -f /opt/airflow/airflow.cfg && exec airflow scheduler"
-    environment:
-      AIRFLOW__CORE__EXECUTOR: LocalExecutor
-      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
-      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
-      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
-      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
-      AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
-      AIRFLOW__LOGGING__WORKER_LOG_SERVER_HOST: airflow-scheduler
-      AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT: "8793"
-      PG_HOST: sc_database
-      PG_PORT: "5432"
-      PG_USER: ${DB_USERNAME}
-      PG_PASSWORD: ${DB_PASSWORD}
-      PG_DATABASE: ${DB_DATABASE_NAME}
-      TYPESENSE_URL: http://typesense:8108
-      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
-    volumes:
-      - airflow_logs:/opt/airflow/logs
-    depends_on:
-      sc_database:
-        condition: service_healthy
-    networks:
-      - backend
-    restart: unless-stopped
-    hostname: airflow-scheduler
-
   # ── Airflow DB Init (one-shot) ───────────────────────────────────────
   airflow-init:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
@@ -282,4 +247,3 @@ volumes:
   kestra_storage:
   supplementary_data:
   typesense_data:
-  airflow_logs: