"""Diagnose the three data gaps blocking the compare-screen redesign.

Run from repo root (network access required, no DB needed):
    uv run --with singer-sdk --with pandas --with requests \
        python pipeline/scripts/diagnose_compare_gaps.py

(singer_sdk is a transitive import of tap_uk_ees.tap / tap_uk_ofsted.tap and
is not part of the repo's default environment, hence the `uv run --with`.)
"""
import io
import re
import sys
import zipfile

import pandas as pd
import requests

sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees")
sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted")
from tap_uk_ees.tap import (  # noqa: E402
    _KS2_NATIONAL_COL_MAP,
    _KS2_NATIONAL_CSV_URL,
    download_release_zip,
    get_all_releases,
)
from tap_uk_ofsted.tap import discover_csv_url  # noqa: E402

TIMEOUT = 120


def check_national_gps_science():
    print("\n=== (a) National catalogue CSV: GPS/science columns ===")
    resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT)
    resp.raise_for_status()
    df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
    df.columns = [c.strip().lower() for c in df.columns]
    for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"):
        status = "PRESENT" if csv_col in df.columns else "MISSING"
        print(f"  {csv_col}: {status}")
    gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c]
    print(f"  all gps/science-ish columns: {gps_like}")
    nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"]
    print(f"  national rows time_periods: {sorted(nat['time_period'].unique())}")
    # Sample the values our map would read for the latest year
    latest = nat[nat["time_period"] == nat["time_period"].max()]
    for csv_col, field in _KS2_NATIONAL_COL_MAP.items():
        val = latest.iloc[0].get(csv_col, "<col missing>") if len(latest) else "<no row>"
        print(f"  {field} <- {csv_col} = {val!r}")


def check_ks2_attainment_years_subjects():
    print("\n=== (b) EES KS2 attainment: years & subject labels ===")
    releases = get_all_releases("key-stage-2-attainment")
    print(f"  releases found: {[r['time_period'] for r in releases]}")
    for release in releases:
        try:
            zf = download_release_zip(release["id"])
        except Exception as e:
            print(f"  {release['time_period']}: DOWNLOAD FAILED: {e}")
            continue
        name = next((n for n in zf.namelist()
                     if "ks2_school_attainment_data" in n and n.endswith(".csv")), None)
        if not name:
            print(f"  {release['time_period']}: NO school attainment CSV in ZIP")
            print(f"    all CSVs in zip: {[n for n in zf.namelist() if n.endswith('.csv')]}")
            continue
        with zf.open(name) as f:
            df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000)
        years = sorted(df["time_period"].unique())
        subjects = sorted(df["subject"].unique())
        print(f"  release {release['time_period']}: time_periods={years}")
        print(f"    subjects={subjects}")


def check_ofsted_report_card_columns():
    print("\n=== (c) Ofsted MI CSV: report-card columns ===")
    url = discover_csv_url()
    print(f"  MI file: {url}")
    if url is None or not url.lower().endswith(".csv"):
        print(f"  URL is not a CSV (likely ODS) — stopping this section. url={url!r}")
        return
    resp = requests.get(url, timeout=TIMEOUT)
    resp.raise_for_status()
    df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5)
    rc_like = [c for c in df.columns
               if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)]
    print(f"  candidate report-card columns ({len(rc_like)}):")
    for c in rc_like:
        print(f"    - {c!r}")
    print(f"  all columns ({len(df.columns)}):")
    for c in df.columns:
        print(f"    - {c!r}")


if __name__ == "__main__":
    check_national_gps_science()
    check_ks2_attainment_years_subjects()
    check_ofsted_report_card_columns()


# FINDINGS 2026-07-12: run via
#   uv run --with singer-sdk --with pandas --with requests \
#       python pipeline/scripts/diagnose_compare_gaps.py
#
# (a) National catalogue CSV (GPS/science) — NOT a source-data problem.
#     pt_gps_exp, pt_scita_exp, avg_readscore, avg_matscore, avg_gpsscore are
#     all PRESENT in the catalogue CSV and hold real numeric values for the
#     latest national row (time_period 202425: pt_gps_exp='72.6' ->
#     gps_expected_pct; pt_scita_exp='81.6' -> science_expected_pct).
#     national time_periods present: 201516, 201617, 201718, 201819, 201920,
#     202021, 202122, 202223, 202324, 202425 (COVID years 201920/202021 are
#     present as rows but suppressed with 'x' per the module docstring, not
#     absent). So _KS2_NATIONAL_COL_MAP is correct and the extractor's own
#     read of the source is fine end-to-end -- the NULLs in
#     marts.fact_ks2_national_averages are NOT caused by a missing/renamed
#     source column. The gap must be introduced downstream of the tap
#     (staging/mart SQL, a stale/incomplete load, or a dbt model not
#     selecting these two columns) -- Task 5/6 should look at the dbt
#     staging model for ees_ks2_national and the mart definition, not the
#     tap/column-map.
#
# (b) EES KS2 attainment (school-level, "key-stage-2-attainment" publication)
#     releases found (via get_all_releases): [None, '202425', '202324',
#     '202223', '202122']. The `None` entry is the *current/latest* release
#     (its slug doesn't parse to a 6-digit time_period by _slug_to_time_period,
#     but the CSV inside carries time_period='202425' -- same data as the
#     202425-labelled release).
#
#     Only two of the four releases contain a school-level attainment CSV
#     matching "ks2_school_attainment_data*.csv":
#       - release None (latest):  HAS IT -> time_periods=['202425']
#           subjects=['Grammar, punctuation and spelling', 'Maths', 'Reading',
#                      'Reading, writing and maths', 'Science', 'Writing']
#       - release 202324:         HAS IT -> time_periods=['202324']
#           subjects= same 6 labels as above
#       - release 202223:         NO school attainment CSV in ZIP. This
#           release's ZIP instead contains only LA/regional/national/MAT-level
#           files (e.g. ks2_regional_and_local_authority_*, ks2_multi_academy
#           _trusts_*, ks2_national_*); no data/*school*attainment*.csv file
#           exists at all in this release's package. This CONFIRMS the
#           "subject-level 2022/23 is NULL in prod" symptom: the source
#           release literally does not publish a school-level attainment file
#           for 202223 under this filename pattern -- it's not a tap bug.
#       - release 202122:         NO school attainment CSV in ZIP. Same
#           situation: ZIP has only LA/regional/national-level files (e.g.
#           ks2_regional_and_local_authority_2016_to_2022_revised.csv,
#           ks2_national_school_characteristics_2016_to_2022_revised.csv);
#           no school-level attainment CSV present. This CONFIRMS "school-level
#           2021/22 is absent" -- again a genuine source-data absence, not an
#           extractor bug.
#     Implication for Tasks 5/6/7: 202122 and 202223 school-level attainment
#     cannot be backfilled from the "key-stage-2-attainment" EES publication
#     via this filename pattern -- those two years must either be sourced
#     from a different EES dataset/file (e.g. one of the *_school_location_
#     and_pupil_characteristics or *_school_type_and_pupil_characteristics
#     files present in those ZIPs, which may carry school-level rows under a
#     different filename), left NULL with an explicit "source unavailable"
#     note, or backfilled from the legacy DfE "Compare School Performance"
#     wide-format CSVs referenced elsewhere in tap.py. Subject labels to use
#     when a source *is* found for 202324/202425:
#       'Grammar, punctuation and spelling', 'Maths', 'Reading',
#       'Reading, writing and maths', 'Science', 'Writing'
#     (Reading, writing and maths spans reading+writing+maths combined --
#     this is the RWM row.)
#
# (c) Ofsted MI CSV (report-card columns) — confirmed PRESENT.
#     discover_csv_url() resolved to (as at run time, latest inspections
#     31 May 2026):
#       https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/
#       Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv
#     This is a real .csv (not .ods) so section (c) ran to completion.
#     Exact report-card column headers (6 grade columns + their paired date
#     columns, all present verbatim, case/spacing exactly as below):
#       'Safeguarding standards'                 / 'Safeguarding standards - date of grade'
#       'Inclusion'                               / 'Inclusion - date of grade'
#       'Curriculum and teaching'                 / 'Curriculum and teaching - date of grade'
#       'Achievement'                             / 'Achievement - date of grade'
#       'Attendance and behaviour'                / 'Attendance and behaviour - date of grade'
#       'Personal development and wellbeing'      / 'Personal development and wellbeing - date of grade'
#       'Leadership and governance'               / 'Leadership and governance - date of grade'
#     Plus a related pass/fail-style field:
#       'Latest OEIF  safeguarding is effective?' (note: double space in the
#       header, verbatim from source -- preserve exactly when mapping)
#     These are the new-style "report card" single-word-area grades
#     (introduced alongside the "Attendance and behaviour" split from
#     "Personal development"); they coexist in the same CSV with the legacy
#     4-judgement OEIF columns ('Latest OEIF overall effectiveness',
#     'Latest OEIF quality of education', 'Latest OEIF behaviour and
#     attitudes', 'Latest OEIF personal development', 'Latest OEIF
#     effectiveness of leadership and management'). Task 7 should map the 7
#     report-card columns above (grade + date pairs, 6 of them, plus the
#     safeguarding-effective flag) rather than inventing new column names.