school_compare/pipeline/scripts/diagnose_compare_gaps.py

"""Diagnose the three data gaps blocking the compare-screen redesign.

Run from repo root (network access required, no DB needed):
    uv run --with singer-sdk --with pandas --with requests \
        python pipeline/scripts/diagnose_compare_gaps.py

(singer_sdk is a transitive import of tap_uk_ees.tap / tap_uk_ofsted.tap and
is not part of the repo's default environment, hence the `uv run --with`.)
"""
import io
import re
import sys

import pandas as pd
import requests

sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees")
sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted")
from tap_uk_ees.tap import (  # noqa: E402
    _KS2_NATIONAL_COL_MAP,
    _KS2_NATIONAL_CSV_URL,
    download_release_zip,
    get_all_releases,
)
from tap_uk_ofsted.tap import discover_csv_url  # noqa: E402

TIMEOUT = 120


def check_national_gps_science():
    print("\n=== (a) National catalogue CSV: GPS/science columns ===")
    resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT)
    resp.raise_for_status()
    df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
    df.columns = [c.strip().lower() for c in df.columns]
    for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"):
        status = "PRESENT" if csv_col in df.columns else "MISSING"
        print(f"  {csv_col}: {status}")
    gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c]
    print(f"  all gps/science-ish columns: {gps_like}")
    if "geographic_level" in df.columns:
        nat = df[df["geographic_level"].str.strip().str.lower() == "national"]
    else:
        print("  geographic_level column missing — cannot isolate national rows")
        return
    print(f"  national rows time_periods: {sorted(nat['time_period'].unique())}")
    # Sample the values our map would read for the latest year
    latest = nat[nat["time_period"] == nat["time_period"].max()]
    for csv_col, field in _KS2_NATIONAL_COL_MAP.items():
        val = latest.iloc[0].get(csv_col, "<col missing>") if len(latest) else "<no row>"
        print(f"  {field} <- {csv_col} = {val!r}")


def check_ks2_attainment_years_subjects():
    print("\n=== (b) EES KS2 attainment: years & subject labels ===")
    releases = get_all_releases("key-stage-2-attainment")
    print(f"  releases found: {[r['time_period'] for r in releases]}")
    for release in releases:
        try:
            zf = download_release_zip(release["id"])
        except Exception as e:
            print(f"  {release['time_period']}: DOWNLOAD FAILED: {e}")
            continue
        name = next((n for n in zf.namelist()
                     if "ks2_school_attainment_data" in n and n.endswith(".csv")), None)
        if not name:
            print(f"  {release['time_period']}: NO school attainment CSV in ZIP")
            print(f"    all CSVs in zip: {[n for n in zf.namelist() if n.endswith('.csv')]}")
            continue
        with zf.open(name) as f:
            df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000)
        years = sorted(df["time_period"].unique())
        subjects = sorted(df["subject"].unique())
        print(f"  release {release['time_period']}: time_periods={years}")
        print(f"    subjects={subjects}")


def check_ofsted_report_card_columns():
    print("\n=== (c) Ofsted MI CSV: report-card columns ===")
    url = discover_csv_url()
    print(f"  MI file: {url}")
    if url is None or not url.lower().endswith(".csv"):
        print(f"  URL is not a CSV (likely ODS) — stopping this section. url={url!r}")
        return
    resp = requests.get(url, timeout=TIMEOUT)
    resp.raise_for_status()
    df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5)
    rc_like = [c for c in df.columns
               if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)]
    print(f"  candidate report-card columns ({len(rc_like)}):")
    for c in rc_like:
        print(f"    - {c!r}")
    print(f"  all columns ({len(df.columns)}):")
    for c in df.columns:
        print(f"    - {c!r}")


if __name__ == "__main__":
    check_national_gps_science()
    check_ks2_attainment_years_subjects()
    check_ofsted_report_card_columns()


# FINDINGS 2026-07-12: run via
#   uv run --with singer-sdk --with pandas --with requests \
#       python pipeline/scripts/diagnose_compare_gaps.py
#
# (a) National catalogue CSV (GPS/science) — NOT a source-data problem.
#     pt_gps_exp, pt_scita_exp, avg_readscore, avg_matscore, avg_gpsscore are
#     all PRESENT in the catalogue CSV and hold real numeric values for the
#     latest national row (time_period 202425: pt_gps_exp='72.6' ->
#     gps_expected_pct; pt_scita_exp='81.6' -> science_expected_pct).
#     national time_periods present: 201516, 201617, 201718, 201819, 201920,
#     202021, 202122, 202223, 202324, 202425 (COVID years 201920/202021 are
#     present as rows but suppressed with 'x' per the module docstring, not
#     absent). So _KS2_NATIONAL_COL_MAP is correct and the extractor's own
#     read of the source is fine end-to-end -- the NULLs in
#     marts.fact_ks2_national_averages are NOT caused by a missing/renamed
#     source column. The gap must be introduced downstream of the tap
#     (staging/mart SQL, a stale/incomplete load, or a dbt model not
#     selecting these two columns) -- Task 5/6 should look at the dbt
#     staging model for ees_ks2_national and the mart definition, not the
#     tap/column-map.
#
# (b) EES KS2 attainment (school-level, "key-stage-2-attainment" publication)
#     releases found (via get_all_releases): [None, '202425', '202324',
#     '202223', '202122']. The `None` entry is the *current/latest* release
#     (its slug doesn't parse to a 6-digit time_period by _slug_to_time_period,
#     but the CSV inside carries time_period='202425' -- same data as the
#     202425-labelled release).
#
#     Only two of the four releases contain a school-level attainment CSV
#     matching "ks2_school_attainment_data*.csv":
#       - release None (latest):  HAS IT -> time_periods=['202425']
#           subjects=['Grammar, punctuation and spelling', 'Maths', 'Reading',
#                      'Reading, writing and maths', 'Science', 'Writing']
#       - release 202324:         HAS IT -> time_periods=['202324']
#           subjects= same 6 labels as above
#       - release 202223:         NO school attainment CSV in ZIP. This
#           release's ZIP instead contains only LA/regional/national/MAT-level
#           files (e.g. ks2_regional_and_local_authority_*, ks2_multi_academy
#           _trusts_*, ks2_national_*); no data/*school*attainment*.csv file
#           exists at all in this release's package. This CONFIRMS the
#           "subject-level 2022/23 is NULL in prod" symptom: the source
#           release literally does not publish a school-level attainment file
#           for 202223 under this filename pattern -- it's not a tap bug.
#       - release 202122:         NO school attainment CSV in ZIP. Same
#           situation: ZIP has only LA/regional/national-level files (e.g.
#           ks2_regional_and_local_authority_2016_to_2022_revised.csv,
#           ks2_national_school_characteristics_2016_to_2022_revised.csv);
#           no school-level attainment CSV present. This CONFIRMS "school-level
#           2021/22 is absent" -- again a genuine source-data absence, not an
#           extractor bug.
#     Implication for Tasks 5/6/7: 202122 and 202223 school-level attainment
#     cannot be backfilled from the "key-stage-2-attainment" EES publication
#     via this filename pattern -- those two years must either be sourced
#     from a different EES dataset/file (e.g. one of the *_school_location_
#     and_pupil_characteristics or *_school_type_and_pupil_characteristics
#     files present in those ZIPs, which may carry school-level rows under a
#     different filename), left NULL with an explicit "source unavailable"
#     note, or backfilled from the legacy DfE "Compare School Performance"
#     wide-format CSVs referenced elsewhere in tap.py. Subject labels to use
#     when a source *is* found for 202324/202425:
#       'Grammar, punctuation and spelling', 'Maths', 'Reading',
#       'Reading, writing and maths', 'Science', 'Writing'
#     (Reading, writing and maths spans reading+writing+maths combined --
#     this is the RWM row.)
#
# (c) Ofsted MI CSV (report-card columns) — confirmed PRESENT.
#     discover_csv_url() resolved to (as at run time, latest inspections
#     31 May 2026):
#       https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/
#       Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv
#     This is a real .csv (not .ods) so section (c) ran to completion.
#     Exact report-card column headers (7 grade columns + their paired date
#     columns, all present verbatim, case/spacing exactly as below):
#       'Safeguarding standards'                 / 'Safeguarding standards - date of grade'
#       'Inclusion'                               / 'Inclusion - date of grade'
#       'Curriculum and teaching'                 / 'Curriculum and teaching - date of grade'
#       'Achievement'                             / 'Achievement - date of grade'
#       'Attendance and behaviour'                / 'Attendance and behaviour - date of grade'
#       'Personal development and wellbeing'      / 'Personal development and wellbeing - date of grade'
#       'Leadership and governance'               / 'Leadership and governance - date of grade'
#     Plus a related pass/fail-style field:
#       'Latest OEIF  safeguarding is effective?' (note: double space in the
#       header, verbatim from source -- preserve exactly when mapping)
#     These are the new-style "report card" single-word-area grades
#     (introduced alongside the "Attendance and behaviour" split from
#     "Personal development"); they coexist in the same CSV with the legacy
#     5-judgement OEIF columns ('Latest OEIF overall effectiveness',
#     'Latest OEIF quality of education', 'Latest OEIF behaviour and
#     attitudes', 'Latest OEIF personal development', 'Latest OEIF
#     effectiveness of leadership and management'). Task 7 should map the 7
#     report-card columns above (grade + date pairs, 6 of them, plus the
#     safeguarding-effective flag) rather than inventing new column names.

# TASK 6 VERIFICATION 2026-07-12: 2021/22 legacy KS2 school-level archive
#
# RESULT: BLOCKED at the source-data level. School-level KS2 attainment for
# academic year 2021/22 was never published anywhere publicly by DfE -- not
# in EES (confirmed by Task 1's finding (b) above), not in the legacy
# "Compare School Performance" download wizard, and not as a standalone
# performance-tables archive/ODS on assets.publishing.service.gov.uk. This
# is a deliberate DfE decision, not a gap in our extraction logic.
#
# Confirming quote (Key stage 2 attainment 2021/22 release notes, via
# https://explore-education-statistics.service.gov.uk/find-statistics/
# key-stage-2-attainment/2021-22):
#   "We will not publish key stage 2 data for academic year 2021/22 in
#   performance tables (also known as Compare School and College
#   Performance)." ... "The Department will, however, still produce the
#   normal suite of key stage 2 accountability measures at school and
#   multi-academy trust level and share these securely with primary
#   schools, academy trusts and local authorities to inform school
#   improvement discussions."
# (i.e. school-level 202122 KS2 results exist internally at DfE but were
# withheld from every public channel: performance tables/CSCP, EES, and by
# extension the legacy DfE archives the current legacy_ks2_urls entries in
# meltano.yml were sourced from.)
#
# What was tried:
#  1. Direct download URL pattern from the task brief:
#       https://www.compare-school-performance.service.gov.uk/download-data?download=true&regions=0&filters=KS2&fileformat=csv&year=2021-2022&meta=false
#     -> HTTP 404, HTML error page (not a CSV/ZIP). Saved response inspected;
#     confirmed 404 via response headers (`content-type: text/html`).
#  2. Walked the actual multi-step download wizard at
#     https://www.compare-school-performance.service.gov.uk/download-data
#     with a browser User-Agent and a cookie jar, replicating the GET-based
#     form steps: currentstep=year (downloadYear=2021-2022) -> currentstep=
#     region (regiontype=all&la=0) -> currentstep=datatypes. On the final
#     "datatypes" step, the checkbox list for 2021-2022 has NO "ks2" (or
#     "ks2mats") option at all -- only ks4/ks4prov/ks4underlying/ks5* /
#     pupil-destination/absence/census/mats checkboxes are present.
#     Control check: repeating the same wizard walk for downloadYear=
#     2018-2019, 2022-2023 and 2023-2024 shows a "ks2" (and "ks2mats")
#     checkbox present in all three; downloadYear=2020-2021 (COVID-cancelled
#     KS2 SATs year) also has NO ks2 checkbox, matching the pattern for a
#     year where school-level KS2 genuinely isn't published. 2021-2022
#     behaves identically to the cancelled 2020-2021 year, not like the
#     normal 2018-2019/2022-2023/2023-2024 years.
#  3. Web search for a standalone KS2 2022 performance-tables archive
#     (e.g. "england_ks2final" for 2022) on assets.publishing.service.gov.uk
#     found no such file; only unrelated 2022/2023-dated documents.
#
#  No ZIP was ever obtained -- /tmp/dfe-2021-2022-ks2.zip contains the 404
#  HTML error page from attempt (1) above, not a real archive. It contains
#  no england_ks2final.csv (there is no ZIP to look inside).
#
# Column-map check (brief's Step 1): NOT RUN -- there is no 2021/22
# england_ks2final.csv to check headers against. This is moot until/unless
# a non-public source (e.g. a manual/internal DfE extract) becomes
# available; _LEGACY_KS2_COLUMN_MAP itself is unchanged and untested here.
#
# Recommendation: mark 202122 school-level KS2 as a genuine, permanent
# source-data gap (not a backfill candidate) unless the project can obtain
# the internal DfE extract DfE says it shared "securely with primary
# schools, academy trusts and local authorities" -- that is not a route
# available to this pipeline. Task 6's meltano.yml change (Step 2) and the
# filebrowser upload should NOT proceed for 202122; there is nothing to
# upload.

# TASK 7 VALUE SAMPLE 2026-07-12: live value_counts() over the 7 report-card
# columns (plus the related safeguarding-effective flag) in the same MI CSV
# resolved by discover_csv_url() as at run time (31 May 2026 inspections
# file). Blank cells read as the literal string 'NULL' (matches
# keep_default_na=False in tap.py). Observed non-blank values, verbatim:
#
#   'Safeguarding standards':                 'Met' (1319), 'Not met' (10)
#   'Inclusion':                               'Expected standard' (710),
#       'Strong standard' (447), 'Needs attention' (130), 'Exceptional' (23),
#       'Urgent improvement' (19)
#   'Curriculum and teaching':                 'Expected standard' (797),
#       'Needs attention' (287), 'Strong standard' (206),
#       'Urgent improvement' (28), 'Exceptional' (11)
#   'Achievement':                             'Expected standard' (701),
#       'Needs attention' (364), 'Strong standard' (207),
#       'Urgent improvement' (39), 'Exceptional' (18)
#   'Attendance and behaviour':                'Expected standard' (699),
#       'Strong standard' (405), 'Needs attention' (188),
#       'Urgent improvement' (21), 'Exceptional' (16)
#   'Personal development and wellbeing':      'Expected standard' (728),
#       'Strong standard' (504), 'Needs attention' (66), 'Exceptional' (23),
#       'Urgent improvement' (8)
#   'Leadership and governance':               'Expected standard' (813),
#       'Strong standard' (292), 'Needs attention' (172),
#       'Urgent improvement' (34), 'Exceptional' (18)
#   'Latest OEIF  safeguarding is effective?' (note double space, not used by
#   Task 7 -- kept for completeness): 'Yes' (12970), 'No' (96)
#
# So the 6 graded report-card columns share exactly one 5-value vocabulary:
# {'Exceptional', 'Strong standard', 'Expected standard', 'Needs attention',
# 'Urgent improvement'} -- no 'Attention needed' variant was observed
# anywhere, so parse_report_card_grade.sql does NOT need that speculative
# branch from the task brief. 'Safeguarding standards' is a separate
# two-value vocabulary {'Met', 'Not met'}.
#
# Collision check: 'Achievement' matches by EXACT list-membership
# (`candidate in df_columns`, a Python list containment check against the
# full column-name list, not a substring/regex match) against only
# ['Achievement', 'Achievement - date of grade'] -- the date-paired column
# has a different exact string and is never selected. Same check for
# 'Safeguarding standards' found only itself, its own date-of-grade column,
# and the unrelated 'Latest OEIF  safeguarding is effective?' column (not
# mapped to any rc_* field). No legacy OEIF column is accidentally consumed
# by an rc_ mapping.