From 8a2503230f5566e09df381f3a01bcc7e4778d952 Mon Sep 17 00:00:00 2001
From: Tudor <tudor@sitaru.org>
Date: Thu, 26 Mar 2026 17:00:07 +0000
Subject: [PATCH] fix(airflow): split back to separate scheduler and api-server
 containers

Running both in one container caused JWT secret key race conditions.
Separate containers with the same AIRFLOW__CORE__SECRET_KEY env var
ensures both processes use identical JWT signing keys. Shared
airflow_logs volume allows the api-server to read task logs written
by the scheduler.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 docker-compose.portainer.yml | 40 ++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml
index 6f1dd1c..99e39df 100644
--- a/docker-compose.portainer.yml
+++ b/docker-compose.portainer.yml
@@ -184,11 +184,11 @@ services:
       retries: 3
       start_period: 15s
 
-  # ── Airflow (api-server + scheduler in one container) ─────────────────
-  airflow:
+  # ── Airflow API Server + UI ───────────────────────────────────────────
+  airflow-api-server:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
-    container_name: schoolcompare_airflow
-    command: bash -c "airflow config list >/dev/null 2>&1 && airflow scheduler & sleep 3 && exec airflow api-server --port 8080"
+    container_name: schoolcompare_airflow_api
+    command: airflow api-server --port 8080
     ports:
       - "8080:8080"
     environment:
@@ -206,6 +206,8 @@ services:
       PG_DATABASE: ${DB_DATABASE_NAME}
       TYPESENSE_URL: http://typesense:8108
       TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
+    volumes:
+      - airflow_logs:/opt/airflow/logs
     depends_on:
       sc_database:
         condition: service_healthy
@@ -219,6 +221,34 @@ services:
       retries: 5
       start_period: 60s
 
+  # ── Airflow Scheduler ──────────────────────────────────────────────
+  airflow-scheduler:
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
+    container_name: schoolcompare_airflow_scheduler
+    command: airflow scheduler
+    environment:
+      AIRFLOW__CORE__EXECUTOR: LocalExecutor
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
+      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
+      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+      AIRFLOW__CORE__SECRET_KEY: "school-compare-airflow-secret-key-that-is-long-enough-for-sha512-jwt-signing"
+      AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
+      PG_HOST: sc_database
+      PG_PORT: "5432"
+      PG_USER: ${DB_USERNAME}
+      PG_PASSWORD: ${DB_PASSWORD}
+      PG_DATABASE: ${DB_DATABASE_NAME}
+      TYPESENSE_URL: http://typesense:8108
+      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
+    volumes:
+      - airflow_logs:/opt/airflow/logs
+    depends_on:
+      sc_database:
+        condition: service_healthy
+    networks:
+      - backend
+    restart: unless-stopped
+
   # ── Airflow DB Init (one-shot) ───────────────────────────────────────
   airflow-init:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
@@ -229,6 +259,7 @@ services:
       AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
       AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
       AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+      AIRFLOW__CORE__SECRET_KEY: "school-compare-airflow-secret-key-that-is-long-enough-for-sha512-jwt-signing"
     depends_on:
       sc_database:
         condition: service_healthy
@@ -248,3 +279,4 @@ volumes:
   kestra_storage:
   supplementary_data:
   typesense_data:
+  airflow_logs: