From 0fe1ea0d6a647ad9ada536d2482c511d2c6e5aa3 Mon Sep 17 00:00:00 2001 From: Tudor Date: Sun, 12 Jul 2026 21:25:00 +0100 Subject: [PATCH] chore(pipeline): diagnostic for compare-screen data gaps --- pipeline/scripts/diagnose_compare_gaps.py | 192 ++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 pipeline/scripts/diagnose_compare_gaps.py diff --git a/pipeline/scripts/diagnose_compare_gaps.py b/pipeline/scripts/diagnose_compare_gaps.py new file mode 100644 index 0000000..145a113 --- /dev/null +++ b/pipeline/scripts/diagnose_compare_gaps.py @@ -0,0 +1,192 @@ +"""Diagnose the three data gaps blocking the compare-screen redesign. + +Run from repo root (network access required, no DB needed): + uv run --with singer-sdk --with pandas --with requests \ + python pipeline/scripts/diagnose_compare_gaps.py + +(singer_sdk is a transitive import of tap_uk_ees.tap / tap_uk_ofsted.tap and +is not part of the repo's default environment, hence the `uv run --with`.) +""" +import io +import re +import sys +import zipfile + +import pandas as pd +import requests + +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees") +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted") +from tap_uk_ees.tap import ( # noqa: E402 + _KS2_NATIONAL_COL_MAP, + _KS2_NATIONAL_CSV_URL, + download_release_zip, + get_all_releases, +) +from tap_uk_ofsted.tap import discover_csv_url # noqa: E402 + +TIMEOUT = 120 + + +def check_national_gps_science(): + print("\n=== (a) National catalogue CSV: GPS/science columns ===") + resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False) + df.columns = [c.strip().lower() for c in df.columns] + for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"): + status = "PRESENT" if csv_col in df.columns else "MISSING" + print(f" {csv_col}: {status}") + gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c] + print(f" all gps/science-ish columns: {gps_like}") + nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"] + print(f" national rows time_periods: {sorted(nat['time_period'].unique())}") + # Sample the values our map would read for the latest year + latest = nat[nat["time_period"] == nat["time_period"].max()] + for csv_col, field in _KS2_NATIONAL_COL_MAP.items(): + val = latest.iloc[0].get(csv_col, "") if len(latest) else "" + print(f" {field} <- {csv_col} = {val!r}") + + +def check_ks2_attainment_years_subjects(): + print("\n=== (b) EES KS2 attainment: years & subject labels ===") + releases = get_all_releases("key-stage-2-attainment") + print(f" releases found: {[r['time_period'] for r in releases]}") + for release in releases: + try: + zf = download_release_zip(release["id"]) + except Exception as e: + print(f" {release['time_period']}: DOWNLOAD FAILED: {e}") + continue + name = next((n for n in zf.namelist() + if "ks2_school_attainment_data" in n and n.endswith(".csv")), None) + if not name: + print(f" {release['time_period']}: NO school attainment CSV in ZIP") + print(f" all CSVs in zip: {[n for n in zf.namelist() if n.endswith('.csv')]}") + continue + with zf.open(name) as f: + df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000) + years = sorted(df["time_period"].unique()) + subjects = sorted(df["subject"].unique()) + print(f" release {release['time_period']}: time_periods={years}") + print(f" subjects={subjects}") + + +def check_ofsted_report_card_columns(): + print("\n=== (c) Ofsted MI CSV: report-card columns ===") + url = discover_csv_url() + print(f" MI file: {url}") + if url is None or not url.lower().endswith(".csv"): + print(f" URL is not a CSV (likely ODS) — stopping this section. url={url!r}") + return + resp = requests.get(url, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5) + rc_like = [c for c in df.columns + if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)] + print(f" candidate report-card columns ({len(rc_like)}):") + for c in rc_like: + print(f" - {c!r}") + print(f" all columns ({len(df.columns)}):") + for c in df.columns: + print(f" - {c!r}") + + +if __name__ == "__main__": + check_national_gps_science() + check_ks2_attainment_years_subjects() + check_ofsted_report_card_columns() + + +# FINDINGS 2026-07-12: run via +# uv run --with singer-sdk --with pandas --with requests \ +# python pipeline/scripts/diagnose_compare_gaps.py +# +# (a) National catalogue CSV (GPS/science) — NOT a source-data problem. +# pt_gps_exp, pt_scita_exp, avg_readscore, avg_matscore, avg_gpsscore are +# all PRESENT in the catalogue CSV and hold real numeric values for the +# latest national row (time_period 202425: pt_gps_exp='72.6' -> +# gps_expected_pct; pt_scita_exp='81.6' -> science_expected_pct). +# national time_periods present: 201516, 201617, 201718, 201819, 201920, +# 202021, 202122, 202223, 202324, 202425 (COVID years 201920/202021 are +# present as rows but suppressed with 'x' per the module docstring, not +# absent). So _KS2_NATIONAL_COL_MAP is correct and the extractor's own +# read of the source is fine end-to-end -- the NULLs in +# marts.fact_ks2_national_averages are NOT caused by a missing/renamed +# source column. The gap must be introduced downstream of the tap +# (staging/mart SQL, a stale/incomplete load, or a dbt model not +# selecting these two columns) -- Task 5/6 should look at the dbt +# staging model for ees_ks2_national and the mart definition, not the +# tap/column-map. +# +# (b) EES KS2 attainment (school-level, "key-stage-2-attainment" publication) +# releases found (via get_all_releases): [None, '202425', '202324', +# '202223', '202122']. The `None` entry is the *current/latest* release +# (its slug doesn't parse to a 6-digit time_period by _slug_to_time_period, +# but the CSV inside carries time_period='202425' -- same data as the +# 202425-labelled release). +# +# Only two of the four releases contain a school-level attainment CSV +# matching "ks2_school_attainment_data*.csv": +# - release None (latest): HAS IT -> time_periods=['202425'] +# subjects=['Grammar, punctuation and spelling', 'Maths', 'Reading', +# 'Reading, writing and maths', 'Science', 'Writing'] +# - release 202324: HAS IT -> time_periods=['202324'] +# subjects= same 6 labels as above +# - release 202223: NO school attainment CSV in ZIP. This +# release's ZIP instead contains only LA/regional/national/MAT-level +# files (e.g. ks2_regional_and_local_authority_*, ks2_multi_academy +# _trusts_*, ks2_national_*); no data/*school*attainment*.csv file +# exists at all in this release's package. This CONFIRMS the +# "subject-level 2022/23 is NULL in prod" symptom: the source +# release literally does not publish a school-level attainment file +# for 202223 under this filename pattern -- it's not a tap bug. +# - release 202122: NO school attainment CSV in ZIP. Same +# situation: ZIP has only LA/regional/national-level files (e.g. +# ks2_regional_and_local_authority_2016_to_2022_revised.csv, +# ks2_national_school_characteristics_2016_to_2022_revised.csv); +# no school-level attainment CSV present. This CONFIRMS "school-level +# 2021/22 is absent" -- again a genuine source-data absence, not an +# extractor bug. +# Implication for Tasks 5/6/7: 202122 and 202223 school-level attainment +# cannot be backfilled from the "key-stage-2-attainment" EES publication +# via this filename pattern -- those two years must either be sourced +# from a different EES dataset/file (e.g. one of the *_school_location_ +# and_pupil_characteristics or *_school_type_and_pupil_characteristics +# files present in those ZIPs, which may carry school-level rows under a +# different filename), left NULL with an explicit "source unavailable" +# note, or backfilled from the legacy DfE "Compare School Performance" +# wide-format CSVs referenced elsewhere in tap.py. Subject labels to use +# when a source *is* found for 202324/202425: +# 'Grammar, punctuation and spelling', 'Maths', 'Reading', +# 'Reading, writing and maths', 'Science', 'Writing' +# (Reading, writing and maths spans reading+writing+maths combined -- +# this is the RWM row.) +# +# (c) Ofsted MI CSV (report-card columns) — confirmed PRESENT. +# discover_csv_url() resolved to (as at run time, latest inspections +# 31 May 2026): +# https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/ +# Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv +# This is a real .csv (not .ods) so section (c) ran to completion. +# Exact report-card column headers (6 grade columns + their paired date +# columns, all present verbatim, case/spacing exactly as below): +# 'Safeguarding standards' / 'Safeguarding standards - date of grade' +# 'Inclusion' / 'Inclusion - date of grade' +# 'Curriculum and teaching' / 'Curriculum and teaching - date of grade' +# 'Achievement' / 'Achievement - date of grade' +# 'Attendance and behaviour' / 'Attendance and behaviour - date of grade' +# 'Personal development and wellbeing' / 'Personal development and wellbeing - date of grade' +# 'Leadership and governance' / 'Leadership and governance - date of grade' +# Plus a related pass/fail-style field: +# 'Latest OEIF safeguarding is effective?' (note: double space in the +# header, verbatim from source -- preserve exactly when mapping) +# These are the new-style "report card" single-word-area grades +# (introduced alongside the "Attendance and behaviour" split from +# "Personal development"); they coexist in the same CSV with the legacy +# 4-judgement OEIF columns ('Latest OEIF overall effectiveness', +# 'Latest OEIF quality of education', 'Latest OEIF behaviour and +# attitudes', 'Latest OEIF personal development', 'Latest OEIF +# effectiveness of leadership and management'). Task 7 should map the 7 +# report-card columns above (grade + date pairs, 6 of them, plus the +# safeguarding-effective flag) rather than inventing new column names.