From d677b5453365b72c81d6df2de62b1fa0d05d684d Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 9 Jul 2026 21:26:45 +0100 Subject: [PATCH] fix(pipeline): cache invalidation for IDACI DAG too; curl timeouts Addresses AI-review findings: the annual IDACI DAG also rebuilds a mart (fact_deprivation) and needs the reload; curl gets connect/max timeouts so an unreachable backend fails fast instead of hanging the task. Co-Authored-By: Claude Fable 5 --- pipeline/dags/school_data_pipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 79529ab..02addb1 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -49,6 +49,7 @@ ADMIN_KEY="${ADMIN_API_KEY:-changeme}" echo "Calling $BACKEND_URL/api/admin/reload ..." response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\ + --connect-timeout 10 --max-time 120 \\ -X POST "$BACKEND_URL/api/admin/reload" \\ -H "X-API-Key: $ADMIN_KEY" \\ -H "Content-Type: application/json") @@ -217,4 +218,9 @@ with DAG( bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_idaci+ fact_deprivation+", ) - extract_idaci >> dbt_build_idaci + invalidate_cache_idaci = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_idaci >> dbt_build_idaci >> invalidate_cache_idaci