From c353e360725826b76dafd5a4a43e89fc9844faac Mon Sep 17 00:00:00 2001
From: Tudor <tudor@Tudors-MacBook-Pro.local>
Date: Thu, 9 Jul 2026 21:12:01 +0100
Subject: [PATCH 1/2] fix(pipeline): actually invalidate the backend cache
 after data rebuilds
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The daily/monthly/annual DAG docstring promised an Invalidate Cache step
that never existed — after a marts rebuild the backend kept serving its
startup-cached (possibly empty) DataFrame until a container restart.
Add a POST /api/admin/reload task at the end of each pipeline DAG,
mirroring the sitemap DAG's admin-call pattern.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 pipeline/dags/school_data_pipeline.py | 45 +++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 3 deletions(-)

diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py
index b8f85da..79529ab 100644
--- a/pipeline/dags/school_data_pipeline.py
+++ b/pipeline/dags/school_data_pipeline.py
@@ -38,6 +38,30 @@ default_args = {
     "retry_delay": timedelta(minutes=5),
 }
 
+# The backend caches the marts DataFrame at startup; after any rebuild the
+# cache must be invalidated or the API serves stale (or empty) data until the
+# container restarts.
+INVALIDATE_CACHE_CMD = """
+set -e
+BACKEND_URL="${BACKEND_URL:-http://backend:80}"
+ADMIN_KEY="${ADMIN_API_KEY:-changeme}"
+
+echo "Calling $BACKEND_URL/api/admin/reload ..."
+
+response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\
+    -X POST "$BACKEND_URL/api/admin/reload" \\
+    -H "X-API-Key: $ADMIN_KEY" \\
+    -H "Content-Type: application/json")
+
+echo "HTTP status: $response"
+cat /tmp/reload_response.json
+
+if [ "$response" != "200" ]; then
+    echo "ERROR: backend cache reload failed (HTTP $response)"
+    exit 1
+fi
+"""
+
 
 # ── Daily DAG (GIAS + downstream) ──────────────────────────────────────
 
@@ -91,7 +115,12 @@ print(f'Validation passed: {{count}} GIAS rows')
         bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py",
     )
 
-    extract_group >> validate_raw >> dbt_build >> sync_typesense
+    invalidate_cache = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_group >> validate_raw >> dbt_build >> sync_typesense >> invalidate_cache
 
 
 # ── Monthly DAG (Ofsted) ───────────────────────────────────────────────
@@ -121,7 +150,12 @@ with DAG(
         bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py",
     )
 
-    extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted
+    invalidate_cache_ofsted = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted >> invalidate_cache_ofsted
 
 
 # ── Annual DAG (EES: KS2, KS4, Census, Admissions) ───────────────────
@@ -153,7 +187,12 @@ with DAG(
         bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py",
     )
 
-    extract_ees_group >> dbt_build_ees >> sync_typesense_ees
+    invalidate_cache_ees = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_ees_group >> dbt_build_ees >> sync_typesense_ees >> invalidate_cache_ees
 
 
 # ── Annual DAG (IDACI Deprivation) ────────────────────────────────────
-- 
2.54.0


From d677b5453365b72c81d6df2de62b1fa0d05d684d Mon Sep 17 00:00:00 2001
From: Tudor <tudor@Tudors-MacBook-Pro.local>
Date: Thu, 9 Jul 2026 21:26:45 +0100
Subject: [PATCH 2/2] fix(pipeline): cache invalidation for IDACI DAG too; curl
 timeouts

Addresses AI-review findings: the annual IDACI DAG also rebuilds a mart
(fact_deprivation) and needs the reload; curl gets connect/max timeouts
so an unreachable backend fails fast instead of hanging the task.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 pipeline/dags/school_data_pipeline.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py
index 79529ab..02addb1 100644
--- a/pipeline/dags/school_data_pipeline.py
+++ b/pipeline/dags/school_data_pipeline.py
@@ -49,6 +49,7 @@ ADMIN_KEY="${ADMIN_API_KEY:-changeme}"
 echo "Calling $BACKEND_URL/api/admin/reload ..."
 
 response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\
+    --connect-timeout 10 --max-time 120 \\
     -X POST "$BACKEND_URL/api/admin/reload" \\
     -H "X-API-Key: $ADMIN_KEY" \\
     -H "Content-Type: application/json")
@@ -217,4 +218,9 @@ with DAG(
         bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_idaci+ fact_deprivation+",
     )
 
-    extract_idaci >> dbt_build_idaci
+    invalidate_cache_idaci = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_idaci >> dbt_build_idaci >> invalidate_cache_idaci
-- 
2.54.0