From 03cd1de6af0ea172c016c14609eb1a8d9cb45cae Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 26 Mar 2026 21:28:03 +0000 Subject: [PATCH] fix(airflow): delete and reimport DAGs on init to clear stale task refs When tasks are removed from a DAG, old serialized metadata in the DB causes 'Task not found' errors. Delete all DAGs before reserializing on each deploy to ensure a clean state. Co-Authored-By: Claude Opus 4.6 --- docker-compose.portainer.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml index bf7aaf2..5dabfa6 100644 --- a/docker-compose.portainer.yml +++ b/docker-compose.portainer.yml @@ -257,7 +257,7 @@ services: airflow-init: image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest container_name: schoolcompare_airflow_init - command: bash -c "airflow db migrate && airflow dags reserialize" + command: bash -c "airflow db migrate && airflow dags delete school_data_daily -y 2>/dev/null; airflow dags delete school_data_monthly_ofsted -y 2>/dev/null; airflow dags delete school_data_annual_ees -y 2>/dev/null; airflow dags reserialize" environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME} diff --git a/docker-compose.yml b/docker-compose.yml index d3f36e3..85c67cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -152,7 +152,7 @@ services: airflow-init: image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest container_name: schoolcompare_airflow_init - command: bash -c "airflow db migrate && airflow dags reserialize" + command: bash -c "airflow db migrate && airflow dags delete school_data_daily -y 2>/dev/null; airflow dags delete school_data_monthly_ofsted -y 2>/dev/null; airflow dags delete school_data_annual_ees -y 2>/dev/null; airflow dags reserialize" environment: *airflow-env depends_on: db: