fix(pipeline): harden banding against EES sentinels; diagnostic cleanups

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB

pipeline/scripts/diagnose_compare_gaps.py

@@ -10,7 +10,6 @@ is not part of the repo's default environment, hence the `uv run --with`.)
 import io
 import re
 import sys
-import zipfile
 
 import pandas as pd
 import requests
@@ -39,7 +38,11 @@ def check_national_gps_science():
         print(f"  {csv_col}: {status}")
     gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c]
     print(f"  all gps/science-ish columns: {gps_like}")
-    nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"]
+    if "geographic_level" in df.columns:
+        nat = df[df["geographic_level"].str.strip().str.lower() == "national"]
+    else:
+        print("  geographic_level column missing — cannot isolate national rows")
+        return
     print(f"  national rows time_periods: {sorted(nat['time_period'].unique())}")
     # Sample the values our map would read for the latest year
     latest = nat[nat["time_period"] == nat["time_period"].max()]
@@ -169,7 +172,7 @@ if __name__ == "__main__":
 #       https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/
 #       Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv
 #     This is a real .csv (not .ods) so section (c) ran to completion.
-#     Exact report-card column headers (6 grade columns + their paired date
+#     Exact report-card column headers (7 grade columns + their paired date
 #     columns, all present verbatim, case/spacing exactly as below):
 #       'Safeguarding standards'                 / 'Safeguarding standards - date of grade'
 #       'Inclusion'                               / 'Inclusion - date of grade'
@@ -184,7 +187,7 @@ if __name__ == "__main__":
 #     These are the new-style "report card" single-word-area grades
 #     (introduced alongside the "Attendance and behaviour" split from
 #     "Personal development"); they coexist in the same CSV with the legacy
-#     4-judgement OEIF columns ('Latest OEIF overall effectiveness',
+#     5-judgement OEIF columns ('Latest OEIF overall effectiveness',
 #     'Latest OEIF quality of education', 'Latest OEIF behaviour and
 #     attitudes', 'Latest OEIF personal development', 'Latest OEIF
 #     effectiveness of leadership and management'). Task 7 should map the 7

pipeline/transform/models/marts/_marts_schema.yml

@@ -110,6 +110,7 @@ models:
               values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average']
               config:
                 where: "progress_8_banding is not null"
+                severity: warn
       - name: attainment_8_disadvantage_gap
       - name: progress_8_disadvantage_gap
     tests:

pipeline/transform/models/staging/stg_ees_ks4.sql

@@ -63,7 +63,13 @@ info as (
         {{ safe_numeric('sen_pupil_percent') }} as sen_pct,
         {{ safe_numeric('sen_with_ehcp_pupil_percent') }} as sen_ehcp_pct,
         {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct,
-        nullif(trim(progress8_banding), '') as progress_8_banding,
+        -- EES suppression sentinels (z/c/x/q/u) and blanks must not reach the
+        -- mart as banding labels
+        case
+            when lower(trim(progress8_banding)) in ('', 'z', 'c', 'x', 'q', 'u', 'null')
+                then null
+            else trim(progress8_banding)
+        end as progress_8_banding,
         {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap,
         {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap
     from {{ source('raw', 'ees_ks4_info') }}