fix(pipeline): migrate to Airflow 3 API server and SimpleAuthManager

Airflow 3 replaced `airflow webserver` with `airflow api-server` and
removed the `airflow users` CLI. Auth is now via SimpleAuthManager
configured through AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS env var.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

docker-compose.portainer.yml

@@ -7,7 +7,7 @@
 #   ADMIN_API_KEY          — Backend admin API key
 #   TYPESENSE_API_KEY      — Typesense admin API key
 #   TYPESENSE_SEARCH_KEY   — Typesense search-only key (exposed to frontend)
-#   AIRFLOW_ADMIN_PASSWORD — Airflow web UI admin password
+#   AIRFLOW_ADMIN_USER    — Airflow admin username (password auto-generated, see api-server logs)
 #   KESTRA_USER            — Kestra UI username (optional)
 #   KESTRA_PASSWORD        — Kestra UI password (optional)
 
@@ -184,11 +184,11 @@ services:
       retries: 3
       start_period: 15s
 
-  # ── Airflow Webserver (UI at :8080) ──────────────────────────────────
-  airflow-webserver:
+  # ── Airflow API Server + UI (at :8080) ────────────────────────────────
+  airflow-api-server:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
-    container_name: schoolcompare_airflow_webserver
-    command: airflow webserver --port 8080
+    container_name: schoolcompare_airflow_api
+    command: airflow api-server --port 8080
     ports:
       - "8080:8080"
     environment:
@@ -196,7 +196,7 @@ services:
       AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
       AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
       AIRFLOW__CORE__LOAD_EXAMPLES: "false"
-      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "false"
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
       PG_HOST: sc_database
       PG_PORT: "5432"
       PG_USER: ${DB_USERNAME}
@@ -229,7 +229,7 @@ services:
       AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
       AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
       AIRFLOW__CORE__LOAD_EXAMPLES: "false"
-      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "false"
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
       PG_HOST: sc_database
       PG_PORT: "5432"
       PG_USER: ${DB_USERNAME}
@@ -250,23 +250,12 @@ services:
   airflow-init:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
     container_name: schoolcompare_airflow_init
-    command: >
-      bash -c "
-        airflow db migrate &&
-        airflow users create
-          --username admin
-          --password $${AIRFLOW_ADMIN_PASSWORD:-admin}
-          --firstname Admin
-          --lastname User
-          --role Admin
-          --email admin@localhost || true
-      "
+    command: airflow db migrate
     environment:
       AIRFLOW__CORE__EXECUTOR: LocalExecutor
       AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
       AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
       AIRFLOW__CORE__LOAD_EXAMPLES: "false"
-      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-admin}
     depends_on:
       sc_database:
         condition: service_healthy

docker-compose.yml

@@ -96,11 +96,11 @@ services:
       retries: 5
       start_period: 10s
 
-  # Apache Airflow — workflow orchestrator (UI at http://localhost:8080)
-  airflow-webserver:
+  # Apache Airflow — API server + UI (http://localhost:8080)
+  airflow-api-server:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
-    container_name: schoolcompare_airflow_webserver
-    command: airflow webserver --port 8080
+    container_name: schoolcompare_airflow_api
+    command: airflow api-server --port 8080
     ports:
       - "8080:8080"
     environment: &airflow-env
@@ -108,7 +108,7 @@ services:
       AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://schoolcompare:schoolcompare@db:5432/schoolcompare
       AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
       AIRFLOW__CORE__LOAD_EXAMPLES: "false"
-      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "false"
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "admin:admin"
       PG_HOST: db
       PG_PORT: "5432"
       PG_USER: schoolcompare
@@ -149,11 +149,7 @@ services:
   airflow-init:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
     container_name: schoolcompare_airflow_init
-    command: >
-      bash -c "
-        airflow db migrate &&
-        airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@localhost || true
-      "
+    command: airflow db migrate
     environment: *airflow-env
     depends_on:
       db:

pipeline/Dockerfile

@@ -34,4 +34,4 @@ RUN cd transform && dbt deps --profiles-dir . 2>/dev/null || true
 ENV AIRFLOW_HOME=/opt/airflow
 ENV PYTHONPATH=/opt/pipeline
 
-CMD ["airflow", "webserver"]
+CMD ["airflow", "api-server"]