193 lines
10 KiB
Python
193 lines
10 KiB
Python
"""Diagnose the three data gaps blocking the compare-screen redesign.
|
|
|
|
Run from repo root (network access required, no DB needed):
|
|
uv run --with singer-sdk --with pandas --with requests \
|
|
python pipeline/scripts/diagnose_compare_gaps.py
|
|
|
|
(singer_sdk is a transitive import of tap_uk_ees.tap / tap_uk_ofsted.tap and
|
|
is not part of the repo's default environment, hence the `uv run --with`.)
|
|
"""
|
|
import io
|
|
import re
|
|
import sys
|
|
import zipfile
|
|
|
|
import pandas as pd
|
|
import requests
|
|
|
|
sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees")
|
|
sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted")
|
|
from tap_uk_ees.tap import ( # noqa: E402
|
|
_KS2_NATIONAL_COL_MAP,
|
|
_KS2_NATIONAL_CSV_URL,
|
|
download_release_zip,
|
|
get_all_releases,
|
|
)
|
|
from tap_uk_ofsted.tap import discover_csv_url # noqa: E402
|
|
|
|
TIMEOUT = 120
|
|
|
|
|
|
def check_national_gps_science():
|
|
print("\n=== (a) National catalogue CSV: GPS/science columns ===")
|
|
resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT)
|
|
resp.raise_for_status()
|
|
df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
|
|
df.columns = [c.strip().lower() for c in df.columns]
|
|
for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"):
|
|
status = "PRESENT" if csv_col in df.columns else "MISSING"
|
|
print(f" {csv_col}: {status}")
|
|
gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c]
|
|
print(f" all gps/science-ish columns: {gps_like}")
|
|
nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"]
|
|
print(f" national rows time_periods: {sorted(nat['time_period'].unique())}")
|
|
# Sample the values our map would read for the latest year
|
|
latest = nat[nat["time_period"] == nat["time_period"].max()]
|
|
for csv_col, field in _KS2_NATIONAL_COL_MAP.items():
|
|
val = latest.iloc[0].get(csv_col, "<col missing>") if len(latest) else "<no row>"
|
|
print(f" {field} <- {csv_col} = {val!r}")
|
|
|
|
|
|
def check_ks2_attainment_years_subjects():
|
|
print("\n=== (b) EES KS2 attainment: years & subject labels ===")
|
|
releases = get_all_releases("key-stage-2-attainment")
|
|
print(f" releases found: {[r['time_period'] for r in releases]}")
|
|
for release in releases:
|
|
try:
|
|
zf = download_release_zip(release["id"])
|
|
except Exception as e:
|
|
print(f" {release['time_period']}: DOWNLOAD FAILED: {e}")
|
|
continue
|
|
name = next((n for n in zf.namelist()
|
|
if "ks2_school_attainment_data" in n and n.endswith(".csv")), None)
|
|
if not name:
|
|
print(f" {release['time_period']}: NO school attainment CSV in ZIP")
|
|
print(f" all CSVs in zip: {[n for n in zf.namelist() if n.endswith('.csv')]}")
|
|
continue
|
|
with zf.open(name) as f:
|
|
df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000)
|
|
years = sorted(df["time_period"].unique())
|
|
subjects = sorted(df["subject"].unique())
|
|
print(f" release {release['time_period']}: time_periods={years}")
|
|
print(f" subjects={subjects}")
|
|
|
|
|
|
def check_ofsted_report_card_columns():
|
|
print("\n=== (c) Ofsted MI CSV: report-card columns ===")
|
|
url = discover_csv_url()
|
|
print(f" MI file: {url}")
|
|
if url is None or not url.lower().endswith(".csv"):
|
|
print(f" URL is not a CSV (likely ODS) — stopping this section. url={url!r}")
|
|
return
|
|
resp = requests.get(url, timeout=TIMEOUT)
|
|
resp.raise_for_status()
|
|
df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5)
|
|
rc_like = [c for c in df.columns
|
|
if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)]
|
|
print(f" candidate report-card columns ({len(rc_like)}):")
|
|
for c in rc_like:
|
|
print(f" - {c!r}")
|
|
print(f" all columns ({len(df.columns)}):")
|
|
for c in df.columns:
|
|
print(f" - {c!r}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
check_national_gps_science()
|
|
check_ks2_attainment_years_subjects()
|
|
check_ofsted_report_card_columns()
|
|
|
|
|
|
# FINDINGS 2026-07-12: run via
|
|
# uv run --with singer-sdk --with pandas --with requests \
|
|
# python pipeline/scripts/diagnose_compare_gaps.py
|
|
#
|
|
# (a) National catalogue CSV (GPS/science) — NOT a source-data problem.
|
|
# pt_gps_exp, pt_scita_exp, avg_readscore, avg_matscore, avg_gpsscore are
|
|
# all PRESENT in the catalogue CSV and hold real numeric values for the
|
|
# latest national row (time_period 202425: pt_gps_exp='72.6' ->
|
|
# gps_expected_pct; pt_scita_exp='81.6' -> science_expected_pct).
|
|
# national time_periods present: 201516, 201617, 201718, 201819, 201920,
|
|
# 202021, 202122, 202223, 202324, 202425 (COVID years 201920/202021 are
|
|
# present as rows but suppressed with 'x' per the module docstring, not
|
|
# absent). So _KS2_NATIONAL_COL_MAP is correct and the extractor's own
|
|
# read of the source is fine end-to-end -- the NULLs in
|
|
# marts.fact_ks2_national_averages are NOT caused by a missing/renamed
|
|
# source column. The gap must be introduced downstream of the tap
|
|
# (staging/mart SQL, a stale/incomplete load, or a dbt model not
|
|
# selecting these two columns) -- Task 5/6 should look at the dbt
|
|
# staging model for ees_ks2_national and the mart definition, not the
|
|
# tap/column-map.
|
|
#
|
|
# (b) EES KS2 attainment (school-level, "key-stage-2-attainment" publication)
|
|
# releases found (via get_all_releases): [None, '202425', '202324',
|
|
# '202223', '202122']. The `None` entry is the *current/latest* release
|
|
# (its slug doesn't parse to a 6-digit time_period by _slug_to_time_period,
|
|
# but the CSV inside carries time_period='202425' -- same data as the
|
|
# 202425-labelled release).
|
|
#
|
|
# Only two of the four releases contain a school-level attainment CSV
|
|
# matching "ks2_school_attainment_data*.csv":
|
|
# - release None (latest): HAS IT -> time_periods=['202425']
|
|
# subjects=['Grammar, punctuation and spelling', 'Maths', 'Reading',
|
|
# 'Reading, writing and maths', 'Science', 'Writing']
|
|
# - release 202324: HAS IT -> time_periods=['202324']
|
|
# subjects= same 6 labels as above
|
|
# - release 202223: NO school attainment CSV in ZIP. This
|
|
# release's ZIP instead contains only LA/regional/national/MAT-level
|
|
# files (e.g. ks2_regional_and_local_authority_*, ks2_multi_academy
|
|
# _trusts_*, ks2_national_*); no data/*school*attainment*.csv file
|
|
# exists at all in this release's package. This CONFIRMS the
|
|
# "subject-level 2022/23 is NULL in prod" symptom: the source
|
|
# release literally does not publish a school-level attainment file
|
|
# for 202223 under this filename pattern -- it's not a tap bug.
|
|
# - release 202122: NO school attainment CSV in ZIP. Same
|
|
# situation: ZIP has only LA/regional/national-level files (e.g.
|
|
# ks2_regional_and_local_authority_2016_to_2022_revised.csv,
|
|
# ks2_national_school_characteristics_2016_to_2022_revised.csv);
|
|
# no school-level attainment CSV present. This CONFIRMS "school-level
|
|
# 2021/22 is absent" -- again a genuine source-data absence, not an
|
|
# extractor bug.
|
|
# Implication for Tasks 5/6/7: 202122 and 202223 school-level attainment
|
|
# cannot be backfilled from the "key-stage-2-attainment" EES publication
|
|
# via this filename pattern -- those two years must either be sourced
|
|
# from a different EES dataset/file (e.g. one of the *_school_location_
|
|
# and_pupil_characteristics or *_school_type_and_pupil_characteristics
|
|
# files present in those ZIPs, which may carry school-level rows under a
|
|
# different filename), left NULL with an explicit "source unavailable"
|
|
# note, or backfilled from the legacy DfE "Compare School Performance"
|
|
# wide-format CSVs referenced elsewhere in tap.py. Subject labels to use
|
|
# when a source *is* found for 202324/202425:
|
|
# 'Grammar, punctuation and spelling', 'Maths', 'Reading',
|
|
# 'Reading, writing and maths', 'Science', 'Writing'
|
|
# (Reading, writing and maths spans reading+writing+maths combined --
|
|
# this is the RWM row.)
|
|
#
|
|
# (c) Ofsted MI CSV (report-card columns) — confirmed PRESENT.
|
|
# discover_csv_url() resolved to (as at run time, latest inspections
|
|
# 31 May 2026):
|
|
# https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/
|
|
# Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv
|
|
# This is a real .csv (not .ods) so section (c) ran to completion.
|
|
# Exact report-card column headers (6 grade columns + their paired date
|
|
# columns, all present verbatim, case/spacing exactly as below):
|
|
# 'Safeguarding standards' / 'Safeguarding standards - date of grade'
|
|
# 'Inclusion' / 'Inclusion - date of grade'
|
|
# 'Curriculum and teaching' / 'Curriculum and teaching - date of grade'
|
|
# 'Achievement' / 'Achievement - date of grade'
|
|
# 'Attendance and behaviour' / 'Attendance and behaviour - date of grade'
|
|
# 'Personal development and wellbeing' / 'Personal development and wellbeing - date of grade'
|
|
# 'Leadership and governance' / 'Leadership and governance - date of grade'
|
|
# Plus a related pass/fail-style field:
|
|
# 'Latest OEIF safeguarding is effective?' (note: double space in the
|
|
# header, verbatim from source -- preserve exactly when mapping)
|
|
# These are the new-style "report card" single-word-area grades
|
|
# (introduced alongside the "Attendance and behaviour" split from
|
|
# "Personal development"); they coexist in the same CSV with the legacy
|
|
# 4-judgement OEIF columns ('Latest OEIF overall effectiveness',
|
|
# 'Latest OEIF quality of education', 'Latest OEIF behaviour and
|
|
# attitudes', 'Latest OEIF personal development', 'Latest OEIF
|
|
# effectiveness of leadership and management'). Task 7 should map the 7
|
|
# report-card columns above (grade + date pairs, 6 of them, plus the
|
|
# safeguarding-effective flag) rather than inventing new column names.
|