diff --git a/pipeline/scripts/diagnose_compare_gaps.py b/pipeline/scripts/diagnose_compare_gaps.py index 840effb..123ca99 100644 --- a/pipeline/scripts/diagnose_compare_gaps.py +++ b/pipeline/scripts/diagnose_compare_gaps.py @@ -10,7 +10,6 @@ is not part of the repo's default environment, hence the `uv run --with`.) import io import re import sys -import zipfile import pandas as pd import requests @@ -39,7 +38,11 @@ def check_national_gps_science(): print(f" {csv_col}: {status}") gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c] print(f" all gps/science-ish columns: {gps_like}") - nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"] + if "geographic_level" in df.columns: + nat = df[df["geographic_level"].str.strip().str.lower() == "national"] + else: + print(" geographic_level column missing — cannot isolate national rows") + return print(f" national rows time_periods: {sorted(nat['time_period'].unique())}") # Sample the values our map would read for the latest year latest = nat[nat["time_period"] == nat["time_period"].max()] @@ -169,7 +172,7 @@ if __name__ == "__main__": # https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/ # Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv # This is a real .csv (not .ods) so section (c) ran to completion. -# Exact report-card column headers (6 grade columns + their paired date +# Exact report-card column headers (7 grade columns + their paired date # columns, all present verbatim, case/spacing exactly as below): # 'Safeguarding standards' / 'Safeguarding standards - date of grade' # 'Inclusion' / 'Inclusion - date of grade' @@ -184,7 +187,7 @@ if __name__ == "__main__": # These are the new-style "report card" single-word-area grades # (introduced alongside the "Attendance and behaviour" split from # "Personal development"); they coexist in the same CSV with the legacy -# 4-judgement OEIF columns ('Latest OEIF overall effectiveness', +# 5-judgement OEIF columns ('Latest OEIF overall effectiveness', # 'Latest OEIF quality of education', 'Latest OEIF behaviour and # attitudes', 'Latest OEIF personal development', 'Latest OEIF # effectiveness of leadership and management'). Task 7 should map the 7 diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 58bb7b8..bd4072e 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -110,6 +110,7 @@ models: values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average'] config: where: "progress_8_banding is not null" + severity: warn - name: attainment_8_disadvantage_gap - name: progress_8_disadvantage_gap tests: diff --git a/pipeline/transform/models/staging/stg_ees_ks4.sql b/pipeline/transform/models/staging/stg_ees_ks4.sql index 7395ebc..217b0bf 100644 --- a/pipeline/transform/models/staging/stg_ees_ks4.sql +++ b/pipeline/transform/models/staging/stg_ees_ks4.sql @@ -63,7 +63,13 @@ info as ( {{ safe_numeric('sen_pupil_percent') }} as sen_pct, {{ safe_numeric('sen_with_ehcp_pupil_percent') }} as sen_ehcp_pct, {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct, - nullif(trim(progress8_banding), '') as progress_8_banding, + -- EES suppression sentinels (z/c/x/q/u) and blanks must not reach the + -- mart as banding labels + case + when lower(trim(progress8_banding)) in ('', 'z', 'c', 'x', 'q', 'u', 'null') + then null + else trim(progress8_banding) + end as progress_8_banding, {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap, {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap from {{ source('raw', 'ees_ks4_info') }}