From 4b75152ee0d818875598d01b12a9374e55b677db Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 9 Jul 2026 14:44:27 +0100 Subject: [PATCH 01/59] fix(api): fall back to legacy name-column query when marts predate code migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the deploy window flagged by CI review — the backend now works against both the old (name) and new (code) mart schemas. Co-Authored-By: Claude Fable 5 --- backend/data_loader.py | 63 +++++++++++++++++++++----- backend/tests/test_gias_translation.py | 52 +++++++++++++++++++++ 2 files changed, 104 insertions(+), 11 deletions(-) diff --git a/backend/data_loader.py b/backend/data_loader.py index db72748..be1303d 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -262,25 +262,66 @@ assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), ( "expected replacement of 's.has_sixth_form,' to have taken effect" ) +# Fallback used when marts.dim_school predates the GIAS code-dictionary +# migration (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on +# this DB, so it still has the old name columns instead of *_code columns). +_MAIN_QUERY_LEGACY_NAMES = str(_MAIN_QUERY) +_LEGACY_NAME_REPLACEMENTS = [ + ("s.phase_code,", "s.phase,"), + ("s.school_type_code,", "s.school_type,"), + ( + "s.religious_character_code,", + "s.religious_character AS religious_denomination,", + ), + ("s.status_code,", "s.status,"), + ("s.admissions_policy_code,", "s.admissions_policy,"), +] +for _old, _new in _LEGACY_NAME_REPLACEMENTS: + assert _old in _MAIN_QUERY_LEGACY_NAMES, ( + f"expected {_old!r} to be present in _MAIN_QUERY before replacement" + ) + _MAIN_QUERY_LEGACY_NAMES = _MAIN_QUERY_LEGACY_NAMES.replace(_old, _new) +_MAIN_QUERY_LEGACY_NAMES = text(_MAIN_QUERY_LEGACY_NAMES) + +_GIAS_CODE_COLUMN_NAMES = ( + "phase_code", + "school_type_code", + "religious_character_code", + "status_code", + "admissions_policy_code", +) + def load_school_data_as_dataframe() -> pd.DataFrame: """Load all school + KS2 data as a pandas DataFrame.""" try: df = pd.read_sql(_MAIN_QUERY, engine) except sqlalchemy.exc.ProgrammingError as exc: - if "has_sixth_form" not in str(exc): + if any(col in str(exc) for col in _GIAS_CODE_COLUMN_NAMES): + logging.getLogger(__name__).warning( + "marts predate the GIAS code migration — falling back to " + "legacy name-column query: %s", + exc, + ) + try: + df = pd.read_sql(_MAIN_QUERY_LEGACY_NAMES, engine) + except Exception as exc2: + print(f"Warning: Could not load school data from marts: {exc2}") + return pd.DataFrame() + elif "has_sixth_form" in str(exc): + logging.getLogger(__name__).warning( + "marts.dim_school is missing has_sixth_form (pipeline hasn't " + "rebuilt the mart yet on this DB) — retrying without it: %s", + exc, + ) + try: + df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine) + except Exception as exc2: + print(f"Warning: Could not load school data from marts: {exc2}") + return pd.DataFrame() + else: print(f"Warning: Could not load school data from marts: {exc}") return pd.DataFrame() - logging.getLogger(__name__).warning( - "marts.dim_school is missing has_sixth_form (pipeline hasn't " - "rebuilt the mart yet on this DB) — retrying without it: %s", - exc, - ) - try: - df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine) - except Exception as exc2: - print(f"Warning: Could not load school data from marts: {exc2}") - return pd.DataFrame() except Exception as exc: print(f"Warning: Could not load school data from marts: {exc}") return pd.DataFrame() diff --git a/backend/tests/test_gias_translation.py b/backend/tests/test_gias_translation.py index 7586010..682cff4 100644 --- a/backend/tests/test_gias_translation.py +++ b/backend/tests/test_gias_translation.py @@ -42,3 +42,55 @@ def test_missing_code_columns_are_a_noop(): out = translate_gias_code_columns(df) assert out.iloc[0]["phase"] == "Primary" assert out.iloc[0]["status"] == "Open" + + +def test_load_school_data_survives_premigration_marts(monkeypatch): + """Real prod state until the nightly pipeline first rebuilds the mart with + the GIAS code columns: marts.dim_school still has the old name columns + (phase, school_type, religious_character, status, admissions_policy) + instead of the new *_code columns. The first query raises UndefinedColumn + on s.phase_code; load_school_data_as_dataframe must retry with the + legacy name-column query rather than swallow the error and return (and + then have load_school_data cache) an empty DataFrame.""" + import sqlalchemy.exc + from backend import data_loader + + data_loader._df_cache = None + data_loader._df_latest_cache = None + + good_df = pd.DataFrame( + [ + { + "urn": 1, + "school_name": "Legacy School", + "phase": "Primary", + "school_type": "Academy", + "status": "Open", + } + ] + ) + calls = [] + + def fake_read_sql(query, con): + calls.append(query) + if len(calls) == 1: + raise sqlalchemy.exc.ProgrammingError( + "(psycopg2.errors.UndefinedColumn) column s.phase_code does not exist", + None, + None, + ) + return good_df.copy() + + monkeypatch.setattr(data_loader.pd, "read_sql", fake_read_sql) + + try: + df = data_loader.load_school_data_as_dataframe() + finally: + data_loader._df_cache = None + data_loader._df_latest_cache = None + + assert len(calls) == 2, "must retry with the legacy name-column query variant" + assert calls[1] is data_loader._MAIN_QUERY_LEGACY_NAMES + assert not df.empty + assert df["phase"].iloc[0] == "Primary" + assert df["status"].iloc[0] == "Open" From 74ca76d150deec6725259d9637ea86d7bb90c683 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 9 Jul 2026 19:29:58 +0100 Subject: [PATCH 02/59] fix(api): match missing-column fallbacks on the DBAPI error, not the statement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit str(ProgrammingError) embeds the full SQL, which contains every column name — the substring check matched any error and could take the wrong retry branch. Parse the missing column from exc.orig instead. Co-Authored-By: Claude Fable 5 --- backend/data_loader.py | 20 ++++++++++-- backend/tests/test_gias_translation.py | 44 +++++++++++++++++++++++--- backend/tests/test_sixth_form_flag.py | 10 ++++-- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/backend/data_loader.py b/backend/data_loader.py index be1303d..75c4528 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -4,6 +4,7 @@ Provides efficient queries with caching. """ import logging +import re import pandas as pd import numpy as np @@ -291,13 +292,28 @@ _GIAS_CODE_COLUMN_NAMES = ( "admissions_policy_code", ) +_MISSING_COLUMN_RE = re.compile(r'column "?(?:s\.)?(\w+)"? does not exist') + + +def _missing_column_name(exc: Exception) -> Optional[str]: + """Name of the missing column from a psycopg2 UndefinedColumn error. + + Inspects exc.orig (the DBAPI error), whose message names only the + offending column — str(exc) also embeds the full SQL statement, which + contains every column name and therefore must not be matched against. + """ + orig = getattr(exc, "orig", None) + match = _MISSING_COLUMN_RE.search(str(orig) if orig is not None else str(exc)) + return match.group(1) if match else None + def load_school_data_as_dataframe() -> pd.DataFrame: """Load all school + KS2 data as a pandas DataFrame.""" try: df = pd.read_sql(_MAIN_QUERY, engine) except sqlalchemy.exc.ProgrammingError as exc: - if any(col in str(exc) for col in _GIAS_CODE_COLUMN_NAMES): + missing = _missing_column_name(exc) + if missing in _GIAS_CODE_COLUMN_NAMES: logging.getLogger(__name__).warning( "marts predate the GIAS code migration — falling back to " "legacy name-column query: %s", @@ -308,7 +324,7 @@ def load_school_data_as_dataframe() -> pd.DataFrame: except Exception as exc2: print(f"Warning: Could not load school data from marts: {exc2}") return pd.DataFrame() - elif "has_sixth_form" in str(exc): + elif missing == "has_sixth_form": logging.getLogger(__name__).warning( "marts.dim_school is missing has_sixth_form (pipeline hasn't " "rebuilt the mart yet on this DB) — retrying without it: %s", diff --git a/backend/tests/test_gias_translation.py b/backend/tests/test_gias_translation.py index 682cff4..414b8e4 100644 --- a/backend/tests/test_gias_translation.py +++ b/backend/tests/test_gias_translation.py @@ -4,7 +4,7 @@ rest of the backend sees must carry today's name strings.""" import numpy as np import pandas as pd -from backend.data_loader import translate_gias_code_columns +from backend.data_loader import _missing_column_name, translate_gias_code_columns from backend.gias_codes import ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION @@ -44,6 +44,39 @@ def test_missing_code_columns_are_a_noop(): assert out.iloc[0]["status"] == "Open" +def _fake_exc(orig_message): + """A stand-in for sqlalchemy.exc.ProgrammingError: str(exc) embeds the + full SQL statement (deliberately containing every column name below, to + prove the matcher doesn't fall back to it), while .orig carries the real + DBAPI error message naming only the offending column.""" + exc = Exception( + "SELECT s.phase_code, s.school_type_code, s.religious_character_code, " + "s.status_code, s.admissions_policy_code, s.has_sixth_form FROM ... " + f"[SQL: ...] (Background on this error at: https://...)" + ) + exc.orig = Exception(orig_message) if orig_message is not None else None + return exc + + +def test_missing_column_name_quoted(): + assert _missing_column_name(_fake_exc('column "phase_code" does not exist')) == "phase_code" + + +def test_missing_column_name_unquoted(): + assert _missing_column_name(_fake_exc("column phase_code does not exist")) == "phase_code" + + +def test_missing_column_name_table_prefixed(): + assert ( + _missing_column_name(_fake_exc("column s.has_sixth_form does not exist")) + == "has_sixth_form" + ) + + +def test_missing_column_name_no_match_returns_none(): + assert _missing_column_name(_fake_exc("relation \"marts.dim_school\" does not exist")) is None + + def test_load_school_data_survives_premigration_marts(monkeypatch): """Real prod state until the nightly pipeline first rebuilds the mart with the GIAS code columns: marts.dim_school still has the old name columns @@ -75,9 +108,12 @@ def test_load_school_data_survives_premigration_marts(monkeypatch): calls.append(query) if len(calls) == 1: raise sqlalchemy.exc.ProgrammingError( - "(psycopg2.errors.UndefinedColumn) column s.phase_code does not exist", - None, - None, + statement=str(data_loader._MAIN_QUERY), + params=None, + orig=Exception( + "(psycopg2.errors.UndefinedColumn) column s.phase_code " + "does not exist\nLINE 5: s.phase_code," + ), ) return good_df.copy() diff --git a/backend/tests/test_sixth_form_flag.py b/backend/tests/test_sixth_form_flag.py index af32799..a5a87f2 100644 --- a/backend/tests/test_sixth_form_flag.py +++ b/backend/tests/test_sixth_form_flag.py @@ -148,10 +148,14 @@ def test_load_school_data_survives_missing_has_sixth_form_column(monkeypatch): def fake_read_sql(query, con): calls.append(query) if len(calls) == 1: + # The statement text still contains phase_code, school_type_code, + # etc. (it's the full _MAIN_QUERY SELECT list) — that's exactly + # the collision this test guards against: matching must be done + # against exc.orig (the DBAPI error), not str(exc)/the statement. raise sqlalchemy.exc.ProgrammingError( - "SELECT ...", - None, - Exception( + statement=str(data_loader._MAIN_QUERY), + params=None, + orig=Exception( "(psycopg2.errors.UndefinedColumn) column s.has_sixth_form " "does not exist" ), From c353e360725826b76dafd5a4a43e89fc9844faac Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 9 Jul 2026 21:12:01 +0100 Subject: [PATCH 03/59] fix(pipeline): actually invalidate the backend cache after data rebuilds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daily/monthly/annual DAG docstring promised an Invalidate Cache step that never existed — after a marts rebuild the backend kept serving its startup-cached (possibly empty) DataFrame until a container restart. Add a POST /api/admin/reload task at the end of each pipeline DAG, mirroring the sitemap DAG's admin-call pattern. Co-Authored-By: Claude Fable 5 --- pipeline/dags/school_data_pipeline.py | 45 +++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index b8f85da..79529ab 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -38,6 +38,30 @@ default_args = { "retry_delay": timedelta(minutes=5), } +# The backend caches the marts DataFrame at startup; after any rebuild the +# cache must be invalidated or the API serves stale (or empty) data until the +# container restarts. +INVALIDATE_CACHE_CMD = """ +set -e +BACKEND_URL="${BACKEND_URL:-http://backend:80}" +ADMIN_KEY="${ADMIN_API_KEY:-changeme}" + +echo "Calling $BACKEND_URL/api/admin/reload ..." + +response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\ + -X POST "$BACKEND_URL/api/admin/reload" \\ + -H "X-API-Key: $ADMIN_KEY" \\ + -H "Content-Type: application/json") + +echo "HTTP status: $response" +cat /tmp/reload_response.json + +if [ "$response" != "200" ]; then + echo "ERROR: backend cache reload failed (HTTP $response)" + exit 1 +fi +""" + # ── Daily DAG (GIAS + downstream) ────────────────────────────────────── @@ -91,7 +115,12 @@ print(f'Validation passed: {{count}} GIAS rows') bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py", ) - extract_group >> validate_raw >> dbt_build >> sync_typesense + invalidate_cache = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_group >> validate_raw >> dbt_build >> sync_typesense >> invalidate_cache # ── Monthly DAG (Ofsted) ─────────────────────────────────────────────── @@ -121,7 +150,12 @@ with DAG( bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py", ) - extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted + invalidate_cache_ofsted = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted >> invalidate_cache_ofsted # ── Annual DAG (EES: KS2, KS4, Census, Admissions) ─────────────────── @@ -153,7 +187,12 @@ with DAG( bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py", ) - extract_ees_group >> dbt_build_ees >> sync_typesense_ees + invalidate_cache_ees = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_ees_group >> dbt_build_ees >> sync_typesense_ees >> invalidate_cache_ees # ── Annual DAG (IDACI Deprivation) ──────────────────────────────────── From d677b5453365b72c81d6df2de62b1fa0d05d684d Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 9 Jul 2026 21:26:45 +0100 Subject: [PATCH 04/59] fix(pipeline): cache invalidation for IDACI DAG too; curl timeouts Addresses AI-review findings: the annual IDACI DAG also rebuilds a mart (fact_deprivation) and needs the reload; curl gets connect/max timeouts so an unreachable backend fails fast instead of hanging the task. Co-Authored-By: Claude Fable 5 --- pipeline/dags/school_data_pipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 79529ab..02addb1 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -49,6 +49,7 @@ ADMIN_KEY="${ADMIN_API_KEY:-changeme}" echo "Calling $BACKEND_URL/api/admin/reload ..." response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\ + --connect-timeout 10 --max-time 120 \\ -X POST "$BACKEND_URL/api/admin/reload" \\ -H "X-API-Key: $ADMIN_KEY" \\ -H "Content-Type: application/json") @@ -217,4 +218,9 @@ with DAG( bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_idaci+ fact_deprivation+", ) - extract_idaci >> dbt_build_idaci + invalidate_cache_idaci = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_idaci >> dbt_build_idaci >> invalidate_cache_idaci From 297bdbd12e7a7f6e7a8eb34bd5ff48a730bd0093 Mon Sep 17 00:00:00 2001 From: Tudor Date: Sun, 12 Jul 2026 21:19:53 +0100 Subject: [PATCH 05/59] docs: compare-screen redesign spec, expert review, and data-foundation plan Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../2026-07-12-compare-data-foundation.md | 591 ++++++++++++++++++ ...2026-07-11-compare-screen-expert-review.md | 177 ++++++ ...26-07-11-compare-screen-redesign-design.md | 318 ++++++++++ 3 files changed, 1086 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-12-compare-data-foundation.md create mode 100644 docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md create mode 100644 docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md diff --git a/docs/superpowers/plans/2026-07-12-compare-data-foundation.md b/docs/superpowers/plans/2026-07-12-compare-data-foundation.md new file mode 100644 index 0000000..30a3bb8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-compare-data-foundation.md @@ -0,0 +1,591 @@ +# Compare-Screen Data Foundation (Pipeline PR) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Land every pipeline/dbt change the compare-screen redesign needs (spec §5 + §8 of `docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md`): promote raw-but-unstored fields to marts, close the national-averages gaps, and wire the Ofsted report-card columns. + +**Architecture:** Meltano Singer taps load `raw.*` tables; dbt builds `staging` → `marts` (read-only for the backend). All changes here are additive columns/rows — no breaking changes to existing marts. The full `dbt build` runs on the server via the Airflow DAGs; locally we gate with `dbt parse` (no DB needed) plus network-only diagnostic scripts. + +**Tech Stack:** Python (Singer SDK taps), dbt-postgres ~1.10 (invoked as `python -m dbt.cli.main`), Meltano, PostgreSQL. + +## Global Constraints + +- **No new external sources** (spec §5): only fields already in the `raw` schema or in files the taps already download. The one sanctioned tap change is the Ofsted MI report-card columns (spec §5, §8.4) and the legacy-KS2 year addition (same DfE performance-tables source). +- **Additive only:** never rename or drop existing mart columns; the backend maps them 1:1 in `backend/models.py`. +- **Never push to `main`.** Branch: `feat/compare-data-foundation`; PR checks must pass. +- Backend `models.py` changes belong to the follow-up backend PR, not this one. +- dbt invocation is always `python -m dbt.cli.main` (a bare `dbt` resolves to the wrong binary — see `pipeline/dags/school_data_pipeline.py:27`). +- EES suppression codes `z`/`c`/`x` must go through the `safe_numeric` macro. +- Computed benchmarks (FSM/EAL/SEN medians, disadvantaged national average) are **backend work** (spec §5) — explicitly out of scope here. + +--- + +### Task 0: Create the branch + +**Files:** none + +- [ ] **Step 1:** `git checkout main && git pull && git checkout -b feat/compare-data-foundation` + +--- + +### Task 1: Diagnostics — pin the three unknowns + +The spec flags three facts we must confirm from the actual files before wiring code: (a) why `gps_expected_pct`/`science_expected_pct` are NULL in `marts.fact_ks2_national_averages` despite being mapped end-to-end; (b) what the KS2 attainment long file calls its subjects/years for 2021/22 and 2022/23 (subject-level 2022/23 is NULL in prod; school-level 2021/22 is absent); (c) the exact report-card column headers in the current Ofsted MI CSV. + +**Files:** +- Create: `pipeline/scripts/diagnose_compare_gaps.py` + +**Interfaces:** +- Produces: a printed findings report; Tasks 5, 6, 7 consume the confirmed column/label names. Precedent: `pipeline/scripts/diagnose_ees_ks4.py`. + +- [ ] **Step 1: Write the diagnostic script** + +```python +"""Diagnose the three data gaps blocking the compare-screen redesign. + +Run from repo root (network access required, no DB needed): + python pipeline/scripts/diagnose_compare_gaps.py +""" +import io +import re +import sys +import zipfile + +import pandas as pd +import requests + +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees") +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted") +from tap_uk_ees.tap import ( # noqa: E402 + _KS2_NATIONAL_COL_MAP, + _KS2_NATIONAL_CSV_URL, + download_release_zip, + get_all_releases, +) +from tap_uk_ofsted.tap import discover_csv_url # noqa: E402 + +TIMEOUT = 120 + + +def check_national_gps_science(): + print("\n=== (a) National catalogue CSV: GPS/science columns ===") + resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False) + df.columns = [c.strip().lower() for c in df.columns] + for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"): + status = "PRESENT" if csv_col in df.columns else "MISSING" + print(f" {csv_col}: {status}") + gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c] + print(f" all gps/science-ish columns: {gps_like}") + nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"] + print(f" national rows time_periods: {sorted(nat['time_period'].unique())}") + # Sample the values our map would read for the latest year + latest = nat[nat["time_period"] == nat["time_period"].max()] + for csv_col, field in _KS2_NATIONAL_COL_MAP.items(): + val = latest.iloc[0].get(csv_col, "") if len(latest) else "" + print(f" {field} <- {csv_col} = {val!r}") + + +def check_ks2_attainment_years_subjects(): + print("\n=== (b) EES KS2 attainment: years & subject labels ===") + releases = get_all_releases("key-stage-2-attainment") + print(f" releases found: {[r['time_period'] for r in releases]}") + for release in releases: + zf = download_release_zip(release["id"]) + name = next((n for n in zf.namelist() + if "ks2_school_attainment_data" in n and n.endswith(".csv")), None) + if not name: + print(f" {release['time_period']}: NO school attainment CSV in ZIP") + continue + with zf.open(name) as f: + df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000) + years = sorted(df["time_period"].unique()) + subjects = sorted(df["subject"].unique()) + print(f" release {release['time_period']}: time_periods={years}") + print(f" subjects={subjects}") + + +def check_ofsted_report_card_columns(): + print("\n=== (c) Ofsted MI CSV: report-card columns ===") + url = discover_csv_url() + print(f" MI file: {url}") + resp = requests.get(url, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5) + rc_like = [c for c in df.columns + if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)] + print(f" candidate report-card columns ({len(rc_like)}):") + for c in rc_like: + print(f" - {c!r}") + + +if __name__ == "__main__": + check_national_gps_science() + check_ks2_attainment_years_subjects() + check_ofsted_report_card_columns() +``` + +Note: if `_KS2_NATIONAL_CSV_URL` is named differently in `tap_uk_ees/tap.py` (it is defined near the `_KS2_NATIONAL_COL_MAP` around line ~490), import whatever constant holds the catalogue CSV URL. + +- [ ] **Step 2: Run it and record findings** + +Run: `python pipeline/scripts/diagnose_compare_gaps.py 2>&1 | tee /tmp/compare-gaps-findings.txt` +Expected: three sections printed. Paste the findings as a comment block at the bottom of the script (so they're committed evidence), e.g. `# FINDINGS 2026-07-12: pt_gps_exp MISSING (actual col: ...), 202122 present in release X, rc columns: [...]`. + +- [ ] **Step 3: Commit** + +```bash +git add pipeline/scripts/diagnose_compare_gaps.py +git commit -m "chore(pipeline): diagnostic for compare-screen data gaps" +``` + +--- + +### Task 2: Admissions preference detail → mart + +Staging already extracts `second_preference_offers`, `third_preference_offers`, `total_offers` (`stg_ees_admissions.sql:26-29`) — the mart drops them. The cross-LA fields are declared in the tap (`all_applications_from_another_LA`, `offers_to_applicants_from_another_LA`) but not selected in staging. + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_admissions.sql` (after line 33, in `renamed`) +- Modify: `pipeline/transform/models/marts/fact_admissions.sql` +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_admissions block, ~line 120) + +**Interfaces:** +- Produces mart columns: `total_offers int`, `second_preference_offers int`, `third_preference_offers int`, `cross_la_applications int`, `cross_la_offers int`. The backend PR will map these in `FactAdmissions`. + +- [ ] **Step 1: Add cross-LA columns to staging** + +In `stg_ees_admissions.sql`, after the `first_preference_applications` line (line 33): + +```sql + -- Cross-borough demand: applications naming this school from families + -- living in another local authority, and offers made to them. + {{ safe_numeric('"all_applications_from_another_LA"') }}::integer as cross_la_applications, + {{ safe_numeric('"offers_to_applicants_from_another_LA"') }}::integer as cross_la_offers, +``` + +(Quote the identifiers — the tap emits them with mixed case, same trap as `FSM_eligible_percent`, see the header comment in that file. If `dbt parse` or the DAG run later shows the raw columns are lower-cased in Postgres, drop the double quotes.) + +- [ ] **Step 2: Pass everything through the mart** + +Replace the full select list in `fact_admissions.sql`: + +```sql +-- Mart: School admissions — one row per URN per year + +select + urn, + year, + school_phase, + places_offered, + total_offers, + total_applications, + first_preference_applications, + first_preference_offers, + second_preference_offers, + third_preference_offers, + cross_la_applications, + cross_la_offers, + first_preference_offer_pct, + oversubscription_ratio, + oversubscribed, + admissions_policy +from {{ ref('stg_ees_admissions') }} +``` + +- [ ] **Step 3: Add schema tests** + +In `_marts_schema.yml` under `fact_admissions.columns`, append: + +```yaml + - name: second_preference_offers + - name: third_preference_offers + - name: cross_la_applications + - name: cross_la_offers + - name: total_offers +``` + +- [ ] **Step 4: Parse gate** + +Run: `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` +Expected: `Done.` with no compilation errors. + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_admissions.sql pipeline/transform/models/marts/fact_admissions.sql pipeline/transform/models/marts/_marts_schema.yml +git commit -m "feat(pipeline): admissions preference breakdown and cross-LA demand in marts" +``` + +--- + +### Task 3: KS2 progress confidence intervals + writing working-towards + +The tap already emits `progress_measure_lower_conf_interval`, `progress_measure_upper_conf_interval`, `working_towards_expected_standard_pupil_percent` (tap.py:203-206). The staging pivot drops them. These power the CI-based Above/Average/Below progress chips (spec §8, first-review item on statistical honesty). + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_ks2.sql` (inside the `pivoted` CTE, next to each subject's `progress_measure_score` case, lines ~41/55/72, and in the final select ~lines 145-152) +- Modify: `pipeline/transform/models/marts/fact_ks2_performance.sql` +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_ks2_performance block, ~line 82) + +**Interfaces:** +- Produces mart columns: `reading_progress_lower_ci`, `reading_progress_upper_ci`, `writing_progress_lower_ci`, `writing_progress_upper_ci`, `maths_progress_lower_ci`, `maths_progress_upper_ci` (float), `writing_working_towards_pct` (float). + +- [ ] **Step 1: Add pivot cases in staging** + +After the `reading_progress` case (line ~41), add: + +```sql + max(case when subject = 'Reading' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as reading_progress_lower_ci, + max(case when subject = 'Reading' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as reading_progress_upper_ci, +``` + +After the `writing_progress` case (line ~55), add: + +```sql + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as writing_progress_lower_ci, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as writing_progress_upper_ci, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('working_towards_expected_standard_pupil_percent') }} end) as writing_working_towards_pct, +``` + +After the `maths_progress` case (line ~72), add: + +```sql + max(case when subject = 'Maths' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as maths_progress_lower_ci, + max(case when subject = 'Maths' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as maths_progress_upper_ci, +``` + +Then add the seven new columns to the model's final select (next to the existing `p.reading_progress` / `p.writing_progress` / `p.maths_progress` lines ~145-152): + +```sql + p.reading_progress_lower_ci, + p.reading_progress_upper_ci, + p.writing_progress_lower_ci, + p.writing_progress_upper_ci, + p.writing_working_towards_pct, + p.maths_progress_lower_ci, + p.maths_progress_upper_ci, +``` + +- [ ] **Step 2: Pass through the mart** + +In `fact_ks2_performance.sql`, add the same seven column names to the select list immediately after the existing `maths_progress` line (this mart selects staging columns by name; match the file's existing alias style — if columns are selected bare, add them bare). + +- [ ] **Step 3: Schema tests** + +In `_marts_schema.yml` under `fact_ks2_performance.columns`, append the seven names (no tests beyond presence — values are legitimately NULL for 2023/24+ since progress measures ended with 2022/23, spec §4.3): + +```yaml + - name: reading_progress_lower_ci + - name: reading_progress_upper_ci + - name: writing_progress_lower_ci + - name: writing_progress_upper_ci + - name: writing_working_towards_pct + - name: maths_progress_lower_ci + - name: maths_progress_upper_ci +``` + +- [ ] **Step 4: Parse gate** + +Run: `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` +Expected: `Done.` + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_ks2.sql pipeline/transform/models/marts/fact_ks2_performance.sql pipeline/transform/models/marts/_marts_schema.yml +git commit -m "feat(pipeline): KS2 progress confidence intervals and writing working-towards" +``` + +--- + +### Task 4: KS4 — Progress 8 banding and disadvantage gaps + +The tap's `ees_ks4_info` stream already declares `progress8_banding` (DfE's own "well above average … well below average" label — the ready-made secondary chip), `attainment8_diffn` and `progress8_diffn` (tap.py:338-340). Wire them through staging into the mart. + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_ks4.sql` (the CTE that reads `ees_ks4_info` — the same one that already surfaces `sen_pct`; add three columns to its select and to the final joined select) +- Modify: `pipeline/transform/models/marts/fact_ks4_performance.sql` (add after `progress_8_upper_ci`) +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_ks4_performance block, ~line 93) + +**Interfaces:** +- Produces mart columns: `progress_8_banding text`, `attainment_8_disadvantage_gap float`, `progress_8_disadvantage_gap float`. + +- [ ] **Step 1: Staging — select from the info source** + +In the info CTE of `stg_ees_ks4.sql` add: + +```sql + nullif(trim(progress8_banding), '') as progress_8_banding, + {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap, + {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap, +``` + +and add the three names to the model's final select (aliased the same way the CTE's other columns are). + +- [ ] **Step 2: Mart passthrough** + +In `fact_ks4_performance.sql`, after the `progress_8_upper_ci,` line: + +```sql + progress_8_banding, + attainment_8_disadvantage_gap, + progress_8_disadvantage_gap, +``` + +- [ ] **Step 3: Schema tests** — append the three names under `fact_ks4_performance.columns`, plus an accepted-values guard that tolerates NULL: + +```yaml + - name: progress_8_banding + tests: + - accepted_values: + values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average'] + config: + where: "progress_8_banding is not null" + - name: attainment_8_disadvantage_gap + - name: progress_8_disadvantage_gap +``` + +(If the DAG run later shows different capitalisation in the data, fix the accepted values to match the data, not vice versa.) + +- [ ] **Step 4: Parse gate** — `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` → `Done.` + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_ks4.sql pipeline/transform/models/marts/fact_ks4_performance.sql pipeline/transform/models/marts/_marts_schema.yml +git commit -m "feat(pipeline): Progress 8 banding and KS4 disadvantage gaps in marts" +``` + +--- + +### Task 5: National averages — 2015/16 row and GPS/science/scaled-score fix + +Two changes. (1) `stg_ees_ks2_national.sql:34` filters `>= 201617`, which is exactly why the England line starts a year late (2015/16 RWM = 53% exists in the catalogue). (2) GPS/science expected are NULL in prod despite full end-to-end mapping — Task 1's findings say whether the catalogue CSV column names differ from `_KS2_NATIONAL_COL_MAP` (`pt_gps_exp`, `pt_scita_exp`) or whether values are suppressed at source. + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_ks2_national.sql:34` +- Modify (conditional on Task 1 findings): `pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py` (`_KS2_NATIONAL_COL_MAP`) + +**Interfaces:** +- Produces: a 201516 row in `marts.fact_ks2_national_averages`; non-NULL `gps_expected_pct`, `science_expected_pct`, `reading_avg_score`, `maths_avg_score`, `gps_avg_score` for years the DfE publishes them. Backend/frontend consume via `/api/national-averages` unchanged (additive year + newly non-NULL fields). + +- [ ] **Step 1: Widen the year filter** + +In `stg_ees_ks2_national.sql`, change line 34: + +```sql + and cast(trim(time_period) as integer) >= 201516 +``` + +(2015/16 was the first year of the current expected-standard tests; nothing earlier is comparable, so keep a floor.) + +- [ ] **Step 2: Fix the column map per Task 1 findings** + +If Task 1 reported the actual CSV column names for GPS/science/scaled scores differ, update `_KS2_NATIONAL_COL_MAP` in `tap.py` accordingly, e.g. (illustrative — use the diagnosed names): + +```python +_KS2_NATIONAL_COL_MAP = { + # ... existing entries ... + "pt_gps_exp": "gps_expected_pct", # replace key with diagnosed name + "pt_scita_exp": "science_expected_pct", # replace key with diagnosed name +} +``` + +If Task 1 showed the columns are present but suppressed (`x`) at national level for all years, instead delete the two entries from the map, delete the corresponding lines from `stg_ees_ks2_national.sql` and `fact_ks2_national_averages.sql`, and record in the PR description that GPS/science England ticks stay "not in dataset" (the mockups already carry that caveat). + +- [ ] **Step 3: Parse gate** — `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` → `Done.` + +- [ ] **Step 4: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_ks2_national.sql pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py +git commit -m "fix(pipeline): include 2015/16 national averages; fix GPS/science national mapping" +``` + +--- + +### Task 6: Legacy KS2 — load the 2021/22 school-level year + +School-level 2021/22 exists in DfE performance-tables archives (same source as the four legacy years already loaded) but in neither our legacy config (stops at 201819, `pipeline/meltano.yml:33-37`) nor EES (starts 2022/23) — unless Task 1's finding (b) showed an EES release carrying 202122, in which case skip this task and note why in the PR. + +The legacy URLs point at the self-hosted filebrowser (`10.0.1.224:8081`) — **the 2021/22 DfE archive must be uploaded there first; this is the one human dependency in this plan.** + +**Files:** +- Modify: `pipeline/meltano.yml` (legacy_ks2_urls block, line ~33) + +**Interfaces:** +- Produces: `raw.legacy_ks2` rows with `year = '202122'`, flowing through `stg_legacy_ks2` → `fact_ks2_performance` unchanged (the stream maps old column names already; 2021/22 CSVs use the same `PTRWM_EXP`-style headers as 2018/19). + +- [ ] **Step 1: Verify the 2021/22 CSV headers match `_LEGACY_KS2_COLUMN_MAP`** + +Download the DfE 2021/22 KS2 revised archive (gov.uk "Compare School Performance data download": 2021-2022 all-schools ZIP), then: + +Run: `python -c "import zipfile,io,pandas as pd; zf=zipfile.ZipFile('/path/to/2021-2022.zip'); n=[x for x in zf.namelist() if 'ks2final' in x.lower() and x.endswith('.csv')][0]; df=pd.read_csv(zf.open(n), dtype=str, nrows=5); import sys; sys.path.insert(0,'pipeline/plugins/extractors/tap-uk-ees'); from tap_uk_ees.tap import _LEGACY_KS2_COLUMN_MAP as m; missing=[c for c in m if c not in df.columns]; print('missing legacy columns:', missing)"` +Expected: `missing legacy columns: []` (progress columns `READPROG` etc. may legitimately be missing/blank in 2021/22 — acceptable, they load as NULL). + +- [ ] **Step 2: Upload the archive to the filebrowser and add the config entry** + +In `pipeline/meltano.yml` under `legacy_ks2_urls`, add (with the real share URL from the filebrowser upload): + +```yaml + "202122": "http://10.0.1.224:8081/filebrowser/api/public/dl/?inline=true" +``` + +- [ ] **Step 3: Commit** + +```bash +git add pipeline/meltano.yml +git commit -m "feat(pipeline): load 2021/22 school-level KS2 from legacy performance tables" +``` + +- [ ] **Step 4 (only if Task 1(b) showed 2022/23 subject labels differ):** widen the subject matchers in `stg_ees_ks2.sql` the same way GPS already is (`subject ilike '%grammar%' or subject = 'GPS'`), e.g. `subject in ('Reading', 'reading')` → use the diagnosed labels. Parse-gate and commit as `fix(pipeline): match 2022/23 KS2 subject labels`. + +--- + +### Task 7: Ofsted report-card columns (rc_*) + +Resolves the tap TODO (`stg_ofsted_inspections.sql:37`). The marts/backed columns already exist as stubs; this wires real values. Uses Task 1(c)'s confirmed MI column names — the candidates below follow the MI file's existing naming style and must be corrected against the diagnostic output. + +**Files:** +- Modify: `pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py` (COLUMN_PRIORITY ~line 19-72, schema ~line 100-114) +- Create: `pipeline/transform/macros/parse_report_card_grade.sql` +- Modify: `pipeline/transform/models/staging/stg_ofsted_inspections.sql:36-46` + +**Interfaces:** +- Produces mart columns (already declared in `fact_ofsted_inspection`): `rc_safeguarding_met boolean`, and `rc_inclusion` … `rc_sixth_form` as integers on the 5-point scale `1=Exceptional, 2=Strong standard, 3=Expected standard, 4=Needs attention/Attention needed, 5=Urgent improvement`. The backend translates codes to labels (same pattern as `gias_codes.py`), verifying wording against Ofsted's published toolkit (spec §8.4). + +- [ ] **Step 1: Add tap column mappings** + +In `COLUMN_PRIORITY` add (replace candidate strings with Task 1(c)'s exact headers — keep them as priority lists so older files degrade to blank): + +```python + "rc_safeguarding_met": ["Report card safeguarding", "Safeguarding"], + "rc_inclusion": ["Report card inclusion", "Inclusion"], + "rc_curriculum_teaching": ["Report card curriculum and teaching", "Curriculum and teaching"], + "rc_achievement": ["Report card achievement", "Achievement"], + "rc_attendance_behaviour": ["Report card attendance and behaviour", "Attendance and behaviour"], + "rc_personal_development": ["Report card personal development and well-being", "Personal development and well-being"], + "rc_leadership_governance": ["Report card leadership and governance", "Leadership and governance"], + "rc_early_years": ["Report card early years", "Early years"], + "rc_sixth_form": ["Report card sixth form", "Sixth form"], +``` + +And in the stream schema (next to `report_url`, ~line 114): + +```python + th.Property("rc_safeguarding_met", th.StringType), + th.Property("rc_inclusion", th.StringType), + th.Property("rc_curriculum_teaching", th.StringType), + th.Property("rc_achievement", th.StringType), + th.Property("rc_attendance_behaviour", th.StringType), + th.Property("rc_personal_development", th.StringType), + th.Property("rc_leadership_governance", th.StringType), + th.Property("rc_early_years", th.StringType), + th.Property("rc_sixth_form", th.StringType), +``` + +- [ ] **Step 2: Write the grade-parsing macro** + +`pipeline/transform/macros/parse_report_card_grade.sql`: + +```sql +{% macro parse_report_card_grade(column_name) %} + case lower(trim(nullif({{ column_name }}, 'NULL'))) + when 'exceptional' then 1 + when 'strong standard' then 2 + when 'expected standard' then 3 + when 'needs attention' then 4 + when 'attention needed' then 4 + when 'urgent improvement' then 5 + end +{% endmacro %} +``` + +- [ ] **Step 3: Wire staging** + +Replace `stg_ofsted_inspections.sql` lines 36-46 (the NULL stubs) with: + +```sql + -- Report Card fields (post-Nov 2025 framework), 5-point scale: + -- 1 Exceptional · 2 Strong standard · 3 Expected standard + -- · 4 Needs attention · 5 Urgent improvement + (lower(trim(nullif(rc_safeguarding_met, 'NULL'))) = 'met') as rc_safeguarding_met, + {{ parse_report_card_grade('rc_inclusion') }}::integer as rc_inclusion, + {{ parse_report_card_grade('rc_curriculum_teaching') }}::integer as rc_curriculum_teaching, + {{ parse_report_card_grade('rc_achievement') }}::integer as rc_achievement, + {{ parse_report_card_grade('rc_attendance_behaviour') }}::integer as rc_attendance_behaviour, + {{ parse_report_card_grade('rc_personal_development') }}::integer as rc_personal_development, + {{ parse_report_card_grade('rc_leadership_governance') }}::integer as rc_leadership_governance, + {{ parse_report_card_grade('rc_early_years') }}::integer as rc_early_years, + {{ parse_report_card_grade('rc_sixth_form') }}::integer as rc_sixth_form, +``` + +Note `rc_safeguarding_met` becomes boolean (NULL when blank) — matching `fact_ofsted_inspection`'s `rc_safeguarding_met` Boolean column. If `fact_ofsted_inspection.sql` casts these columns, align its casts too (inspect that model; it currently passes the text stubs through). + +- [ ] **Step 4: Parse gate + tap smoke test** + +Run: `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` → `Done.` +Run: `python -c "import sys; sys.path.insert(0,'pipeline/plugins/extractors/tap-uk-ofsted'); from tap_uk_ofsted.tap import COLUMN_PRIORITY; assert 'rc_inclusion' in COLUMN_PRIORITY; print('ok')"` → `ok` + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py pipeline/transform/macros/parse_report_card_grade.sql pipeline/transform/models/staging/stg_ofsted_inspections.sql +git commit -m "feat(pipeline): extract Ofsted report-card judgements (rc_* columns)" +``` + +--- + +### Task 8: PR + post-merge verification + +**Files:** none new + +- [ ] **Step 1: Push and open the PR** (Gitea — use the git credential helper + basic-auth API pattern; token-header auth 401s): + +```bash +git push -u origin feat/compare-data-foundation +# then create the PR via the Gitea API with basic auth from `git credential fill` +``` + +PR body: link spec §5/§8, list the new mart columns, note the Task 6 human dependency (filebrowser upload) and the Task 1 findings file. + +- [ ] **Step 2: After merge, verify the DAG run picked everything up** + +The daily/monthly DAGs rebuild the affected models (`pipeline/dags/school_data_pipeline.py`). Spot-check via the public API (production after promotion, staging first at stx.schoolcompare.co.uk — note external /api is broken at the staging proxy, so check staging from the host): + +```bash +# 2015/16 national row exists +curl -sL "https://www.schoolcompare.co.uk/api/national-averages" | python3 -c "import json,sys; d=json.load(sys.stdin); assert any(r['year']==201516 and r['primary'] for r in d['by_year']), '2015/16 missing'; print('201516 ok')" +# 2021/22 school rows exist (Barclay) +curl -sL "https://www.schoolcompare.co.uk/api/schools/138690" | python3 -c "import json,sys; d=json.load(sys.stdin); ys=[r['year'] for r in d['yearly_data']]; assert 202122 in [int(y) for y in ys], ys; print('202122 ok')" +``` + +(The admissions/CI/KS4/rc_* columns aren't API-visible until the backend PR maps them — verify those directly in Postgres from the pipeline host: `select count(*) from marts.fact_admissions where second_preference_offers is not null;` etc.) + +- [ ] **Step 3: Update the spec** — tick off the §5 promotions this PR delivered (edit the spec's promotion list to note "landed in PR #NN") and commit to main via a docs PR or alongside the backend PR. + +--- + +## Out of scope (next plans) + +1. **Backend PR:** map new columns in `backend/models.py`, extend `/api/compare` with supplementary blocks + `national_averages`, computed benchmarks (FSM/EAL/SEN/size medians, disadvantaged national average), CI-based progress banding, report-card label translation (verify against Ofsted toolkit), Ofsted provider-page URLs, graded-vs-ungraded surfacing. +2. **Frontend PR:** rebuild `/compare` per the mockups + e2e journeys (promotion gate). +3. **Separate bug fix:** third school's series not rendering on the current production chart. +4. **Post-v1 (spec):** census ethnicity/young-carer promotion, IDACI display, attendance section, gender-split/absence tier-2 measures. +5. **Already in marts, no work needed:** KS4 EBacc entry/APS, grade 5+ English & maths, Progress 8 CIs — `fact_ks4_performance` carries them today; only the backend needs to expose them. diff --git a/docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md b/docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md new file mode 100644 index 0000000..d67821d --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md @@ -0,0 +1,177 @@ +# Compare Screen Redesign — Expert Data Review + +**Date:** 2026-07-11 +**Reviewer:** subagent briefed as an English education-standards / DfE-Ofsted data expert +**Subject:** desktop + mobile compare mockups and the redesign spec +(`2026-07-11-compare-screen-redesign-design.md`) +**Status:** first-pass must-fixes applied 2026-07-12; second-pass +findings (below) applied 2026-07-12 — mockups + spec §4/§8 updated + +## Must-fix + +1. **COVID gap is wrong and drops a real results year.** KS2 tests were + cancelled 2019/20 and 2020/21 only; they resumed in 2021/22 with + published school-level results (England RWM ≈ 59%). The mockup charts + omit 2021/22 entirely and the tooltip claims no tests were held + 2019/20–2021/22. Fix: add 2021/22 to axis and all series; shrink the + gap band; optionally annotate 2021/22 with DfE's post-pandemic + comparability caution. +2. **Report-card at-a-glance summary miscounts areas.** Detail list has + 4 Strong / 2 Expected / 1 Attention needed + Safeguarding met, but + the summary says "3 areas Expected standard" — it counts safeguarding + as a graded area. Safeguarding is a separate binary judgement and + must be excluded from rating counts. +3. **"Where the offers went" derivation is unsound.** Places − 1st-pref + offers ≠ "second or third choices": the residual can include 4th–6th + preference offers (pan-London scheme) and LA-allocated children who + didn't choose the school; and offers don't necessarily equal PAN. + Use the real 2nd/3rd-preference fields being promoted from + `raw.ees_admissions`; until then drop the row. +4. **Ofsted timeline in the copy is wrong.** Overall grades were + abolished September 2024, not November 2025; Sept 2024–Nov 2025 + inspections kept the four key judgements without an overall grade + (ungraded inspections carried grades forward). Neither mockup shows + the interim regime, which will dominate real comparisons. Fix copy + and add an interim example. +5. **Barclay's "published an overall grade only — no area-by-area + detail" misdescribes inspections.** No inspection type does that; a + 2021 graded inspection necessarily had subgrades — the gap is in our + dataset. If it was an ungraded (s8) inspection, "Outstanding" is a + carried-forward grade and should say so. Fix: "We don't hold + area-by-area detail for this inspection", and distinguish graded vs + ungraded in the data model. + +## Should-fix + +6. Writing is teacher assessment, not a test — "national tests and + teacher assessments"; note TA caveat on the Writing strip. +7. Verify renewed-framework wording against Ofsted's final toolkit: + likely "Needs attention" (not "Attention needed") and "Personal + development and well-being" (which otherwise collides with the + identically-named legacy judgement). Pin every label to the + published toolkit. +8. "Expected standard" now means two things on one page (Ofsted area + rating vs KS2 measure) — disambiguate in tooltips. +9. Disadvantaged row: DfE definition includes looked-after / previously + looked-after children, not just FSM6; benchmark labels inconsistent + across desktop/mobile; subgroup percentages need cohort sizes or a + volatility threshold before chips are attached. +10. "Trend, last 7 years" spans ten years; sparklines render the COVID + gap as equal spacing (the exact defect the audit criticises) and + "Improved: 52% → 87%" endpoint-cherry-picks a volatile series. +11. At-a-glance "Getting a place" uses different metrics per school + (Barclay is also oversubscribed on total preferences but shows a + green chip). Standardise on first-preference success %. Explain the + equal-preference rule; condition "living close by matters" on the + school's actual oversubscription criteria. +12. "457 applications for 180 places" = total preferences at any rank, + not head-to-head applicants; lead with first preferences vs places. + Add offers-vs-final-intake (waiting lists/appeals) caveat. +13. Elmhurst's subgrade list is likely missing Early years provision + (school has a nursery) — possible pipeline gap. +14. "Ofsted rating" label is obsolete post-Sept-2024 — use "Latest + Ofsted inspection"; check whether Oct 2021 is the latest inspection + or merely the latest graded one. +15. SEN: "EHCP plans" is redundant; 28% SEN support often indicates + resourced provision — add a note; England SEN-support ≈ 14%, not 13%. + +## Nice-to-have + +16. Consistent labelling of official DfE vs dataset-computed benchmarks + (and medians shouldn't be called averages inconsistently). +17. England 2015/16 RWM (53%) exists in DfE publications — the null is + a dataset gap; source it or the England line looks broken. +18. "1 in 4 first choices missed out" — actually more than 1 in 4. +19. "1,273 of 1,260 places (full)" is over capacity; capacity figures + are often stale — say "at or above capacity". +20. State the actual suppression rule (DfE: ≤5 pupils suppressed, + small numbers rounded) instead of "a handful". +21. Spec §4.3 progress chips can't exist for displayed years: KS2 + progress ended with 2022/23 (no KS1 baseline) and returns + ~2027/28 with the reception baseline. Make explicit in the spec. + IDACI (spec §4.5) is absent from mockups; if shipped, caveat it + describes pupils' neighbourhoods, not the school. +22. Tooltips should give the official term "first preference" alongside + the plain-English "first choice". + +## Overall assessment (verbatim gist) + +The bones are genuinely good by education-data standards — +England-average anchoring, explicit non-comparability messaging across +Ofsted regimes, refusal to synthesise an overall grade, time-true +x-axis, neutral FSM/EAL framing — better than most commercial +school-comparison sites. But items 1–5 are outright factual errors or +misdescriptions that a well-informed parent or Ofsted would catch; +the admissions section needs the most conceptual work (equal +preference, preferences-vs-applicants, offers-vs-intake). Fix 1–5 +before user testing; the rest fold into the planned PRs. + +--- + +# Second-pass review (2026-07-12) + +Same reviewer, after the must-fixes and the new three-tier metric +exposure model were applied. + +## Verification of first-pass must-fixes + +- **1 (COVID/2021/22): resolved.** Time-true axis, band covers only the + cancelled years, England 58.7% consistent with official figures, + dataset gaps break lines honestly; reading/maths England series all + match published figures; RWM ≤ min(subject) checks pass. +- **2 (report-card count): resolved** — safeguarding excluded, spec §8.2. +- **3 (offers derivation): resolved** — row removed, spec §8.3 bans it. +- **4 (Ofsted timeline): resolved on desktop; mobile omits the interim + regime clause** (see finding 6). +- **5 (Barclay explanation): resolved.** + +## New findings + +1. **Should-fix — scaled-score strip domain contradicts caption.** + Caption says "scaled scores run 80–120", strips render 100–120; + truncated domain exaggerates small gaps and below-100 averages + would fall off the edge. Render 80–120, or caption the 100–120 + window honestly and define below-100 behaviour. +2. **Should-fix — scaled-score England ticks (106/105/105) unsourced.** + Plausible but hand-entered; verify against DfE 2024/25 tables and + add loading official England scaled scores to the pipeline list + (absent from §8.1/§8.6). +3. **Should-fix — "Writing" listed under "Higher standard" in the + picker.** Writing TA outcome is "greater depth" (GDS), never + "higher standard". Label "Writing — greater depth (teacher + assessment)"; tooltip the combined higher-standard composition. +4. Nice — "grammar & punctuation" summary line drops "spelling" (GPS). +5. Nice — science is teacher-assessed (no KS2 test since 2009) and + coarse; tooltip it like writing; reconsider its tier-2 slot. +6. **Should-fix — mobile Ofsted copy skips the interim regime** + (Sept 2024–Nov 2025) that desktop explains. One clause fixes it. +7. **Should-fix — benchmark provenance still inconsistent** (EAL + tooltip unsourced; FSM/disadvantaged chips vs tooltips use three + vocabularies; header note says all England averages are official). + Adopt one house style: official = "England average", computed = + "benchmark / typical state school (our dataset)". Also tighten EAL + definition to census wording ("first language known or believed to + be other than English"). +8. Nice — "community primaries" distance note attached to an academy + (Elmhurst); say "non-faith primaries" or condition on policy field. +9. Nice — "Improving since 2022" → "since 2022/23". +10. Nice — England chart tooltips show decimals; §7 mandates whole + percents. + +## Residual gaps not covered by spec §8 + +11. Spec promises IDACI-in-words, Attendance section, and tier-2 + gender/absence that the mockups never show — mark post-v1 or + demonstrate, so implementation scope is unambiguous. +12. Add official England scaled-score averages to the pipeline task + list. +13. Add the writing/greater-depth terminology rule to §8.7. + +## Verdict + +All must-fixes genuinely resolved; the tier model is conceptually +sound ("no measure is lost", honest dataset-gap breaks, grouped +picker). Remaining issues are contained: one internal contradiction +(80–120 vs 100–120), one provenance inconsistency, one terminology +error (writing/GDS). With findings 1–3 and 6–7 addressed, the data +framing is fit to put in front of parents. diff --git a/docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md b/docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md new file mode 100644 index 0000000..f160c36 --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md @@ -0,0 +1,318 @@ +# Compare Screen Redesign — Audit & Design + +**Date:** 2026-07-11 +**Status:** Draft — awaiting review +**Scope:** `/compare` page (nextjs-app), `/api/compare` endpoint (backend) + +## 1. Audit of the current screen + +The current compare page (`nextjs-app/components/ComparisonView.tsx`) is a +single-metric analyst tool: a ` + + + + + + + + + + + + + + + + + + + + + + + + School lines break where a year isn't in our dataset. + + +
+ + + + +

+ Sources: DfE Compare School Performance (KS2 results), Ofsted inspection outcomes, DfE school admissions data, school census — all from datasets SchoolCompare already collects. England averages for test results are the official DfE national figures; benchmarks for free school meals, language, SEN, school size and disadvantaged pupils' results are computed across all state schools in our dataset. Following DfE practice, figures based on 5 or fewer pupils are suppressed and shown as "no data". This is a static mockup: tooltips and "Add school" are illustrative, and Plumcroft's Ofsted report card is a made-up example of the November 2025 format (its real latest inspection is Good, June 2023) — no school in our dataset has a report card yet. +

+ + + diff --git a/docs/superpowers/specs/mockups/compare-mobile.html b/docs/superpowers/specs/mockups/compare-mobile.html new file mode 100644 index 0000000..fbb73f1 --- /dev/null +++ b/docs/superpowers/specs/mockups/compare-mobile.html @@ -0,0 +1,437 @@ +Compare screen — mobile mockup + + +
+

Mobile mockup — proposed /compare. Mobile-first layout: measures stack vertically with all schools under each, so nothing needs horizontal swiping. Same live data as the desktop mockup.

+ +

Compare schools

+

Anchored against the England average — the grey tick — so you can tell what's typical at a glance.

+ +
+ Barclay + Elmhurst + Plumcroft + + Add +
+ +

At a glance

+

The short version — each measure is explained in its own section below.

+ +
+
Latest Ofsted inspection
+
BarclayOutstandingOlder-style inspection, Oct 2021
+
ElmhurstOutstandingOlder-style inspection, Oct 2021
+
Plumcroft4 areas Strong standard 2 areas Expected Attendance & behaviour: Attention needed illustrativeNew-style report card, Nov 2025 · safeguarding met · full detail in the Ofsted section below
+
+ +
+
Children reaching the expected standard ?
+
England average: 62%
+
Barclay87% Above average
+
Elmhurst92% Above average
+
Plumcroft79% Above average
+
+ +
+
Getting a place
+
Barclay97% of first choices offeredNamed on 457 forms · 180 places
+
Elmhurst73% of first choices offeredNamed on 342 forms · 120 places
+
PlumcroftAll first choices offeredNamed on 185 forms · 80 places
+
+ +

Ofsted inspection

+

Ofsted stopped giving a single overall grade in September 2024 (inspections until November 2025 kept the area-by-area judgements); from November 2025 new inspections produce a report card rating each area of school life (Exceptional · Strong standard · Expected standard · Attention needed · Urgent improvement). A report card and an older grade aren't directly comparable. Ofsted's "Expected standard" rating is unrelated to the KS2 test measure below.

+ +
+
Latest inspection
+
BarclayOutstanding4+ years ago7 Oct 2021 · we don't hold area-by-area detail for this inspection · Ofsted page →
+
ElmhurstOutstanding 4+ years ago 6 Oct 2021 +
+
Quality of educationOutstanding
+
Behaviour & attitudesOutstanding
+
Personal developmentOutstanding
+
Leadership & managementOutstanding
+
+ Ofsted page → +
+
PlumcroftReport card illustrative 14 Nov 2025 +
+
AchievementStrong standard
+
Curriculum & teachingStrong standard
+
Attendance & behaviourAttention needed
+
Personal developmentStrong standard
+
InclusionExpected standard
+
Leadership & governanceStrong standard
+
Early yearsExpected standard
+
SafeguardingMet
+
+ Ofsted page → +
+
+ +

How children do academically

+

End of Year 6 national tests and teacher assessments (2024/25) — writing is teacher-assessed. Each line runs 0–100%; the grey tick is the England average.

+
+
+ More measures — grammar, punctuation & spelling, science, scaled scores +
+

Strips show the 100–120 window of the full 80–120 scaled-score range; 100 is the expected standard (the strip widens if a school averages below it). England ticks for GPS and science aren't in our dataset yet, and the scaled-score ticks are indicative — official DfE figures will be loaded before launch.

+
+
+ +
+
Children from lower-income families ?
+
State-school average: 46%
+
Barclay86% Well above average
+
Elmhurst93% Well above average
+
Plumcroft72% Above average
+
+ +

Getting a place

+

September 2026 entry. "First choice" = families who ranked the school top of their form (officially a "first preference"). Schools never see your ranking — places go by the admission criteria alone. Figures are National Offer Day offers; waiting lists and appeals can change the final intake.

+
+
First-choice families offered a place
+
Barclay97%Named on 457 forms · 180 places
+
Elmhurst73% Over 1 in 4 missed outNamed on 342 forms · 120 places — check the school's admission criteria (for most non-faith primaries, distance decides)
+
Plumcroft100%Named on 185 forms · 80 places · every first choice offered
+
+ +

Who goes there

+

From the latest school census (2025/26). No "right" numbers here — just context.

+
+
Pupils on roll
+
Barclay1,273At or above capacity · much larger than average · girls 51% / boys 49%
+
Elmhurst98098% full · much larger than average · girls 48% / boys 52%
+
Plumcroft1,056At or above capacity · much larger than average · girls 51% / boys 49%
+
+
+
Free school meals ?
+
State-school average: 25% (our dataset)
+
Barclay26% About average
+
Elmhurst25% About average
+
Plumcroft30% A little above
+
+
+
English as an additional language · extra learning support (SEN) ?
+
BarclayEAL 62% · SEN 6%
+
ElmhurstEAL 84% · SEN 8%
+
PlumcroftEAL 20% · SEN 28% SEN well above avg
+
+
+
Basics
+
BarclayAges 3–11 · nursery · no faith · Lion Academy Trust
+
ElmhurstAges 3–11 · nursery · no faith · New Vision Trust
+
PlumcroftAges 3–11 · nursery · no faith · Greenwich council
+
+ +

Explore trends

+

Every measure from the current compare page lives on here, grouped. Three are wired up in this mockup. School lines break where a year isn't in our dataset.

+
+ +
+
+ +
+
+

← swipe the chart →

+ +

+ Sources: DfE Compare School Performance, Ofsted inspection outcomes, DfE admissions data, school census — all from datasets SchoolCompare already collects. England averages for test results are official DfE figures; FSM, language, SEN, size and disadvantaged-pupil benchmarks are computed across state schools in our dataset. Plumcroft's Ofsted report card is a made-up example of the November 2025 format (its real latest inspection is Good, June 2023). Following DfE practice, figures based on 5 or fewer pupils are suppressed and shown as "no data". Static mockup — tooltips and "+ Add" are illustrative. +

+
+ + From 80f057ea5a8d5ec8a5946d3104bfde99257dac6e Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 19:14:55 +0100 Subject: [PATCH 33/59] feat(compare): types for enriched comparison payload Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- nextjs-app/lib/types.ts | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/nextjs-app/lib/types.ts b/nextjs-app/lib/types.ts index c29512c..3634f87 100644 --- a/nextjs-app/lib/types.ts +++ b/nextjs-app/lib/types.ts @@ -99,6 +99,21 @@ export interface OfstedInspection { rc_leadership_governance: number | null; rc_early_years: number | null; rc_sixth_form: number | null; + /** Where the effective overall grade came from: a graded (Section 5) + * inspection, or carried forward from an ungraded (Section 8) outcome. */ + grade_source?: 'graded' | 'ungraded_carried_forward' | null; + /** Renewed-framework (Nov 2025) area judgements, coded + labelled by the + * backend from the live-sampled Ofsted vocabulary. Empty when the school + * has no report-card inspection. Safeguarding is never included here. */ + report_card?: Record; + /** The school's page on ofsted.gov.uk (never a deep report link). */ + ofsted_page_url?: string; + report_url?: string | null; +} + +export interface ReportCardEntry { + code: number; + label: string; } export interface SchoolCensus { @@ -129,6 +144,12 @@ export interface SchoolAdmissions { /** 1st-preference applications per place offered (>1 means oversubscribed). */ oversubscription_ratio?: number | null; oversubscribed: boolean | null; + total_offers?: number | null; + second_preference_offers?: number | null; + third_preference_offers?: number | null; + /** Applications naming this school from families in another LA, and offers to them. */ + cross_la_applications?: number | null; + cross_la_offers?: number | null; } export interface SenDetail { @@ -172,6 +193,22 @@ export interface SchoolResult { school_id: number; year: number; + // Progress confidence intervals + writing working-towards (published for + // years with progress measures, i.e. up to 2022/23) + reading_progress_lower_ci?: number | null; + reading_progress_upper_ci?: number | null; + writing_progress_lower_ci?: number | null; + writing_progress_upper_ci?: number | null; + writing_working_towards_pct?: number | null; + maths_progress_lower_ci?: number | null; + maths_progress_upper_ci?: number | null; + + // KS4 banding and disadvantage gaps + /** DfE's own plain-English Progress 8 label, e.g. "Well above average". */ + progress_8_banding?: string | null; + attainment_8_disadvantage_gap?: number | null; + progress_8_disadvantage_gap?: number | null; + // Pupil numbers total_pupils: number | null; eligible_pupils: number | null; @@ -308,10 +345,47 @@ export interface SchoolDetailsResponse { export interface ComparisonData { school_info: School; yearly_data: SchoolResult[]; + // Supplementary blocks (additive; absent on an old backend) + ofsted?: OfstedInspection | null; + census?: SchoolCensus | null; + admissions?: SchoolAdmissions | null; + admissions_history?: SchoolAdmissions[]; + deprivation?: SchoolDeprivation | null; +} + +export interface BenchmarkBlock { + eal_pct: number | null; + sen_support_pct: number | null; + disadvantaged_pct: number | null; + median_pupils: number | null; + /** Primary only — weighted by cohort size. */ + disadvantaged_rwm_expected_pct?: number | null; +} + +/** Computed from our dataset — NOT official DfE figures. UI copy must say + * "state-school average (computed from our dataset)" (the `source` string). */ +export interface Benchmarks { + source: string; + year: number; + primary: BenchmarkBlock; + secondary: BenchmarkBlock; +} + +export interface NationalAverages { + year: number; + primary: Record; + secondary: Record; + by_year: Array<{ + year: number; + primary: Record; + secondary: Record; + }>; } export interface ComparisonResponse { comparison: Record; + national_averages?: NationalAverages; + benchmarks?: Benchmarks; } export interface RankingItem { From 48ca042b0853aefc6300a0c2bb5c7e03e09511b8 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 19:17:15 +0100 Subject: [PATCH 34/59] feat(compare): comprehension logic (report cards, admissions, verdicts, strips) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- nextjs-app/__tests__/lib/compareLogic.test.ts | 231 +++++++++++++++++ nextjs-app/lib/compareLogic.ts | 244 ++++++++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 nextjs-app/__tests__/lib/compareLogic.test.ts create mode 100644 nextjs-app/lib/compareLogic.ts diff --git a/nextjs-app/__tests__/lib/compareLogic.test.ts b/nextjs-app/__tests__/lib/compareLogic.test.ts new file mode 100644 index 0000000..d181a5d --- /dev/null +++ b/nextjs-app/__tests__/lib/compareLogic.test.ts @@ -0,0 +1,231 @@ +/** + * compareLogic encodes the expert-reviewed comprehension rules for the + * compare screen: report-card summarisation (safeguarding never counted), + * three-regime Ofsted display, one consistent admissions chip metric, + * CI-based progress banding, verdict chips and dot-strip geometry. + */ + +import { + OFSTED_LEGACY_GRADES, + ofstedDisplay, + progressBand, + rcAreaLabel, + stripPositions, + summariseAdmissions, + summariseReportCard, + verdict, +} from '@/lib/compareLogic'; +import type { OfstedInspection, SchoolAdmissions } from '@/lib/types'; + +function ofsted(partial: Partial): OfstedInspection { + return { + framework: null, + inspection_date: null, + inspection_type: null, + overall_effectiveness: null, + quality_of_education: null, + behaviour_attitudes: null, + personal_development: null, + leadership_management: null, + early_years_provision: null, + previous_overall: null, + rc_safeguarding_met: null, + rc_inclusion: null, + rc_curriculum_teaching: null, + rc_achievement: null, + rc_attendance_behaviour: null, + rc_personal_development: null, + rc_leadership_governance: null, + rc_early_years: null, + rc_sixth_form: null, + ...partial, + }; +} + +const REPORT_CARD = { + rc_achievement: { code: 2, label: 'Strong standard' }, + rc_curriculum_teaching: { code: 2, label: 'Strong standard' }, + rc_personal_development: { code: 2, label: 'Strong standard' }, + rc_leadership_governance: { code: 2, label: 'Strong standard' }, + rc_inclusion: { code: 3, label: 'Expected standard' }, + rc_early_years: { code: 3, label: 'Expected standard' }, + rc_attendance_behaviour: { code: 4, label: 'Needs attention' }, +}; + +describe('summariseReportCard', () => { + it('counts graded areas best-first and NAMES problem areas', () => { + const s = summariseReportCard( + ofsted({ report_card: REPORT_CARD, rc_safeguarding_met: true }), + ); + expect(s.counts).toEqual([ + { label: 'Strong standard', count: 4 }, + { label: 'Expected standard', count: 2 }, + ]); + expect(s.problems).toEqual([ + { areaLabel: 'Attendance & behaviour', label: 'Needs attention' }, + ]); + expect(s.safeguarding).toBe('met'); + expect(s.allClear).toBe(false); + }); + + it('never counts safeguarding as a graded area', () => { + const s = summariseReportCard( + ofsted({ + report_card: { rc_achievement: { code: 3, label: 'Expected standard' } }, + rc_safeguarding_met: true, + }), + ); + const total = s.counts.reduce((n, c) => n + c.count, 0); + expect(total).toBe(1); + }); + + it('is allClear when everything is Expected standard or better and safeguarding met', () => { + const s = summariseReportCard( + ofsted({ + report_card: { + rc_achievement: { code: 3, label: 'Expected standard' }, + rc_inclusion: { code: 1, label: 'Exceptional' }, + }, + rc_safeguarding_met: true, + }), + ); + expect(s.allClear).toBe(true); + expect(s.counts[0]).toEqual({ label: 'Exceptional', count: 1 }); + }); + + it('passes labels through from the API — never invents wording', () => { + const s = summariseReportCard( + ofsted({ report_card: { rc_inclusion: { code: 4, label: 'Needs attention' } } }), + ); + expect(JSON.stringify(s)).not.toContain('Attention needed'); + }); +}); + +describe('ofstedDisplay', () => { + it('prefers the report card over any legacy grade', () => { + const d = ofstedDisplay( + ofsted({ overall_effectiveness: 2, report_card: REPORT_CARD }), + ); + expect(d.kind).toBe('report_card'); + }); + + it('distinguishes graded from carried-forward grades', () => { + const graded = ofstedDisplay( + ofsted({ overall_effectiveness: 1, grade_source: 'graded' }), + ); + expect(graded).toMatchObject({ kind: 'graded', gradeLabel: 'Outstanding', carriedForward: false }); + + const carried = ofstedDisplay( + ofsted({ overall_effectiveness: 2, grade_source: 'ungraded_carried_forward' }), + ); + expect(carried).toMatchObject({ kind: 'carried_forward', gradeLabel: 'Good', carriedForward: true }); + }); + + it('handles missing data', () => { + expect(ofstedDisplay(null).kind).toBe('none'); + expect(ofstedDisplay(ofsted({})).kind).toBe('none'); + }); + + it('uses the four legacy grade words', () => { + expect(OFSTED_LEGACY_GRADES).toEqual({ + 1: 'Outstanding', + 2: 'Good', + 3: 'Requires improvement', + 4: 'Inadequate', + }); + }); +}); + +describe('rcAreaLabel', () => { + it('maps rc keys to the mockups’ area labels', () => { + expect(rcAreaLabel('rc_attendance_behaviour')).toBe('Attendance & behaviour'); + expect(rcAreaLabel('rc_curriculum_teaching')).toBe('Curriculum & teaching'); + expect(rcAreaLabel('rc_leadership_governance')).toBe('Leadership & governance'); + }); +}); + +describe('summariseAdmissions', () => { + function admissions(partial: Partial): SchoolAdmissions { + return { + year: 202627, + places_offered: null, + total_applications: null, + first_preference_offer_pct: null, + oversubscribed: null, + ...partial, + }; + } + + it('97% → good chip with the mockup wording', () => { + const s = summariseAdmissions( + admissions({ first_preference_offer_pct: 96.98, total_applications: 457, places_offered: 180 }), + ); + expect(s.chip).toEqual({ tone: 'good', text: '97% of first choices offered' }); + expect(s.interest).toBe('Named on 457 forms · 180 places'); + }); + + it('73% → warn chip "Over 1 in 4 first choices missed out"', () => { + const s = summariseAdmissions(admissions({ first_preference_offer_pct: 73.4 })); + expect(s.chip).toEqual({ tone: 'warn', text: 'Over 1 in 4 first choices missed out' }); + }); + + it('100% → "All first choices offered"', () => { + const s = summariseAdmissions(admissions({ first_preference_offer_pct: 100 })); + expect(s.chip).toEqual({ tone: 'good', text: 'All first choices offered' }); + }); + + it('no data → null chip and interest', () => { + const s = summariseAdmissions(null); + expect(s.chip).toBeNull(); + expect(s.interest).toBeNull(); + }); +}); + +describe('progressBand', () => { + it('CI entirely above zero → above', () => { + expect(progressBand(1.2, 0.4, 2.0)).toBe('above'); + }); + it('CI entirely below zero → below', () => { + expect(progressBand(-1.2, -2.0, -0.4)).toBe('below'); + }); + it('CI straddling zero → average', () => { + expect(progressBand(0.3, -0.5, 1.1)).toBe('average'); + }); + it('missing CI → null (no naive thresholding)', () => { + expect(progressBand(1.2, null, null)).toBeNull(); + expect(progressBand(null, null, null)).toBeNull(); + }); +}); + +describe('verdict', () => { + it('above / close / below with a 2pp tolerance', () => { + expect(verdict(87, 62)).toBe('above'); + expect(verdict(61, 62)).toBe('close'); + expect(verdict(40, 62)).toBe('below'); + }); +}); + +describe('stripPositions', () => { + it('maps a custom domain', () => { + const pts = stripPositions([106], 100, 120); + expect(pts[0].pos).toBe(30); + }); + + it('flips a colliding label above', () => { + const pts = stripPositions([91, 92], 0, 100); + const sorted = [...pts].sort((a, b) => a.value - b.value); + expect(sorted[0].labelAbove).toBe(false); + expect(sorted[1].labelAbove).toBe(true); + }); + + it('skips nulls and keeps school indices', () => { + const pts = stripPositions([50, null, 70], 0, 100); + expect(pts).toHaveLength(2); + expect(pts.map((p) => p.schoolIndex)).toEqual([0, 2]); + }); + + it('clamps out-of-domain values', () => { + const pts = stripPositions([95], 100, 120); + expect(pts[0].pos).toBe(0); + }); +}); diff --git a/nextjs-app/lib/compareLogic.ts b/nextjs-app/lib/compareLogic.ts new file mode 100644 index 0000000..3d4134f --- /dev/null +++ b/nextjs-app/lib/compareLogic.ts @@ -0,0 +1,244 @@ +/** + * Comprehension rules for the compare screen, kept pure and unit-tested. + * + * These encode the expert-review requirements (spec §8 of the compare + * redesign): report-card summaries count graded areas only (safeguarding is + * a separate binary judgement), problem areas are always NAMED rather than + * folded into counts, grade labels pass through from the API (live-sampled + * Ofsted vocabulary — never invented here), admissions chips use one + * consistent metric, and progress bands follow DfE's confidence-interval + * methodology instead of thresholding point estimates. + */ + +import type { OfstedInspection, SchoolAdmissions } from './types'; + +// --------------------------------------------------------------------------- +// Verdicts against an anchor (England average or state-school benchmark) +// --------------------------------------------------------------------------- + +export type Verdict = 'above' | 'close' | 'below'; + +export function verdict(value: number, anchor: number, tolerance = 2): Verdict { + if (value >= anchor + tolerance) return 'above'; + if (value <= anchor - tolerance) return 'below'; + return 'close'; +} + +// --------------------------------------------------------------------------- +// Ofsted — three regimes, one display model +// --------------------------------------------------------------------------- + +export const OFSTED_LEGACY_GRADES: Record = { + 1: 'Outstanding', + 2: 'Good', + 3: 'Requires improvement', + 4: 'Inadequate', +}; + +/** rc_ key → the area label used across the reviewed mockups. */ +const RC_AREA_LABELS: Record = { + rc_inclusion: 'Inclusion', + rc_curriculum_teaching: 'Curriculum & teaching', + rc_achievement: 'Achievement', + rc_attendance_behaviour: 'Attendance & behaviour', + rc_personal_development: 'Personal development', + rc_leadership_governance: 'Leadership & governance', + rc_early_years: 'Early years', + rc_sixth_form: 'Sixth form', +}; + +export function rcAreaLabel(key: string): string { + return RC_AREA_LABELS[key] ?? key; +} + +export interface ReportCardSummary { + /** Graded areas only, grouped by label, best grade first. */ + counts: Array<{ label: string; count: number }>; + /** Areas rated Needs attention / Urgent improvement — always named. */ + problems: Array<{ areaLabel: string; label: string }>; + safeguarding: 'met' | 'not_met' | null; + /** True when every graded area is Expected standard or better and + * safeguarding is not "not met". */ + allClear: boolean; +} + +const PROBLEM_CODES = new Set([4, 5]); + +export function summariseReportCard(ofsted: OfstedInspection): ReportCardSummary { + const entries = Object.entries(ofsted.report_card ?? {}); + const byCode = new Map(); + const problems: ReportCardSummary['problems'] = []; + + for (const [key, entry] of entries) { + if (PROBLEM_CODES.has(entry.code)) { + problems.push({ areaLabel: rcAreaLabel(key), label: entry.label }); + } else { + const existing = byCode.get(entry.code); + if (existing) existing.count += 1; + else byCode.set(entry.code, { label: entry.label, count: 1 }); + } + } + + const counts = [...byCode.entries()] + .sort(([a], [b]) => a - b) + .map(([, v]) => v); + + const safeguarding = + ofsted.rc_safeguarding_met === true + ? 'met' + : ofsted.rc_safeguarding_met === false + ? 'not_met' + : null; + + return { + counts, + problems, + safeguarding, + allClear: entries.length > 0 && problems.length === 0 && safeguarding !== 'not_met', + }; +} + +export type OfstedDisplay = + | { kind: 'none' } + | { kind: 'graded'; grade: number; gradeLabel: string; carriedForward: false } + | { kind: 'carried_forward'; grade: number; gradeLabel: string; carriedForward: true } + | { kind: 'report_card'; summary: ReportCardSummary }; + +export function ofstedDisplay( + ofsted: OfstedInspection | null | undefined, +): OfstedDisplay { + if (!ofsted) return { kind: 'none' }; + + // A report card is the newest inspection format; when present it wins — + // never derive or prefer an overall grade alongside it. + if (ofsted.report_card && Object.keys(ofsted.report_card).length > 0) { + return { kind: 'report_card', summary: summariseReportCard(ofsted) }; + } + + const grade = ofsted.overall_effectiveness; + const gradeLabel = grade != null ? OFSTED_LEGACY_GRADES[grade] : undefined; + if (grade == null || gradeLabel === undefined) return { kind: 'none' }; + + if (ofsted.grade_source === 'ungraded_carried_forward') { + return { kind: 'carried_forward', grade, gradeLabel, carriedForward: true }; + } + return { kind: 'graded', grade, gradeLabel, carriedForward: false }; +} + +// --------------------------------------------------------------------------- +// Admissions — one consistent chip metric (first-preference success) +// --------------------------------------------------------------------------- + +export interface AdmissionsSummary { + firstPrefPct: number | null; + chip: { tone: 'good' | 'warn' | 'neutral'; text: string } | null; + /** e.g. "Named on 457 forms · 180 places" — total preferences at any rank, + * deliberately not phrased as head-to-head applications. */ + interest: string | null; +} + +export function summariseAdmissions( + a: SchoolAdmissions | null | undefined, +): AdmissionsSummary { + if (!a) return { firstPrefPct: null, chip: null, interest: null }; + + const pct = + a.first_preference_offer_pct != null + ? Math.round(a.first_preference_offer_pct) + : null; + + let chip: AdmissionsSummary['chip'] = null; + if (pct != null) { + if (pct >= 100) { + chip = { tone: 'good', text: 'All first choices offered' }; + } else if (pct < 75) { + chip = { tone: 'warn', text: 'Over 1 in 4 first choices missed out' }; + } else { + chip = { tone: pct >= 90 ? 'good' : 'neutral', text: `${pct}% of first choices offered` }; + } + } + + const interest = + a.total_applications != null && a.places_offered != null + ? `Named on ${a.total_applications.toLocaleString('en-GB')} forms · ${a.places_offered.toLocaleString('en-GB')} places` + : null; + + return { firstPrefPct: pct, chip, interest }; +} + +// --------------------------------------------------------------------------- +// Progress bands — DfE confidence-interval methodology +// --------------------------------------------------------------------------- + +export function progressBand( + score: number | null, + lower: number | null, + upper: number | null, +): 'above' | 'average' | 'below' | null { + if (score == null || lower == null || upper == null) return null; + if (lower > 0) return 'above'; + if (upper < 0) return 'below'; + return 'average'; +} + +// --------------------------------------------------------------------------- +// Dot-strip geometry +// --------------------------------------------------------------------------- + +export interface StripPoint { + /** 0–100 percentage position along the track. */ + pos: number; + labelAbove: boolean; + value: number; + schoolIndex: number; +} + +/** Labels within 4% of the domain of a lower neighbour flip above the strip + * (the reviewed mockups' collision nudge). */ +export function stripPositions( + values: Array, + min = 0, + max = 100, +): StripPoint[] { + const span = max - min; + const points = values + .map((value, schoolIndex) => ({ value, schoolIndex })) + .filter((p): p is { value: number; schoolIndex: number } => p.value != null) + .map((p) => ({ + value: p.value, + schoolIndex: p.schoolIndex, + pos: Math.min(100, Math.max(0, ((p.value - min) / span) * 100)), + labelAbove: false, + })); + + const nudge = span * 0.04; + let lastBelow = -Infinity; + for (const p of [...points].sort((a, b) => a.value - b.value)) { + if (p.value - lastBelow < nudge) { + p.labelAbove = true; + } else { + lastBelow = p.value; + } + } + return points; +} + +// --------------------------------------------------------------------------- +// Metric extraction +// --------------------------------------------------------------------------- + +/** Latest non-null yearly value of `metricKey` per school, in `urns` order. */ +export function latestValues( + data: Record & { year: number }> }>, + urns: number[], + metricKey: string, +): Array { + return urns.map((urn) => { + const rows = data[String(urn)]?.yearly_data ?? []; + for (let i = rows.length - 1; i >= 0; i--) { + const v = rows[i][metricKey]; + if (typeof v === 'number' && !Number.isNaN(v)) return v; + } + return null; + }); +} From 60cbc3f46dcd35da9d3f51e15a77bb5e08ef5b29 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 19:18:33 +0100 Subject: [PATCH 35/59] feat(compare): DotStrip with England-average anchor Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../__tests__/components/DotStrip.test.tsx | 48 +++++++++ nextjs-app/components/DotStrip.module.css | 102 ++++++++++++++++++ nextjs-app/components/DotStrip.tsx | 97 +++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 nextjs-app/__tests__/components/DotStrip.test.tsx create mode 100644 nextjs-app/components/DotStrip.module.css create mode 100644 nextjs-app/components/DotStrip.tsx diff --git a/nextjs-app/__tests__/components/DotStrip.test.tsx b/nextjs-app/__tests__/components/DotStrip.test.tsx new file mode 100644 index 0000000..786f183 --- /dev/null +++ b/nextjs-app/__tests__/components/DotStrip.test.tsx @@ -0,0 +1,48 @@ +import { render, screen } from '@testing-library/react'; + +import { DotStrip } from '@/components/DotStrip'; + +describe('DotStrip', () => { + it('enumerates anchor and school values in the aria-label', () => { + render( + , + ); + const strip = screen.getByRole('img'); + expect(strip).toHaveAccessibleName( + 'Reading: England 75%, Barclay 91%, Elmhurst 92%, Plumcroft 87%', + ); + }); + + it('renders the anchor tick when provided and not otherwise', () => { + const { rerender } = render( + , + ); + expect(screen.getByText('England 75%')).toBeInTheDocument(); + + rerender( + , + ); + expect(screen.queryByText(/England/)).not.toBeInTheDocument(); + }); + + it('skips schools without a value', () => { + render( + , + ); + expect(screen.getByRole('img')).toHaveAccessibleName('Maths: Barclay 91%'); + }); +}); diff --git a/nextjs-app/components/DotStrip.module.css b/nextjs-app/components/DotStrip.module.css new file mode 100644 index 0000000..2b7f380 --- /dev/null +++ b/nextjs-app/components/DotStrip.module.css @@ -0,0 +1,102 @@ +.row { + margin: 1.1rem 0 1.6rem; +} + +.head { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 1rem; + flex-wrap: wrap; +} + +.title { + font-weight: 600; + font-size: 0.95rem; +} + +.headNote { + font-size: 0.8rem; + color: var(--text-muted); +} + +.strip { + position: relative; + height: 34px; + margin-top: 0.45rem; +} + +.track { + position: absolute; + left: 0; + right: 0; + top: 15px; + height: 4px; + border-radius: 2px; + background: var(--bg-secondary); +} + +.anchorTick { + position: absolute; + top: 4px; + width: 2px; + height: 26px; + background: var(--text-muted); +} + +.anchorLabel { + position: absolute; + top: -14px; + transform: translateX(-50%); + font-size: 0.7rem; + color: var(--text-muted); + white-space: nowrap; +} + +.point { + position: absolute; + top: 9px; + width: 16px; + height: 16px; + border-radius: 50%; + transform: translateX(-50%); + border: 2px solid var(--bg-card); + box-shadow: 0 0 0 1px rgba(26, 22, 18, 0.08); +} + +.pointLabel { + position: absolute; + top: 27px; + transform: translateX(-50%); + font-size: 0.72rem; + font-weight: 600; + font-variant-numeric: tabular-nums; + color: var(--text-secondary); +} + +.pointLabelAbove { + top: -6px; +} + +@media (max-width: 760px) { + .row { + margin: 0.9rem 0 1.3rem; + } + + .title { + font-size: 0.82rem; + } + + .strip { + height: 32px; + } + + .point { + width: 14px; + height: 14px; + } + + .pointLabel { + font-size: 0.64rem; + } +} diff --git a/nextjs-app/components/DotStrip.tsx b/nextjs-app/components/DotStrip.tsx new file mode 100644 index 0000000..cd3b843 --- /dev/null +++ b/nextjs-app/components/DotStrip.tsx @@ -0,0 +1,97 @@ +/** + * DotStrip — the compare screen's signature element: one measure per strip, + * every school's dot on a shared track, anchored by a grey England-average + * tick so "right of the tick = above average" needs no domain knowledge. + */ + +'use client'; + +import { stripPositions } from '@/lib/compareLogic'; +import { CHART_COLORS, CHART_TEXT_COLORS } from '@/lib/utils'; +import styles from './DotStrip.module.css'; + +export interface DotStripProps { + label: string; + /** One value per school; index = the school's chart-colour index. */ + values: Array; + schoolNames: string[]; + /** Anchor tick, e.g. { value: 62, label: 'England 62%' }. Omit when the + * benchmark isn't available — the caller should say why in `headNote`. */ + anchor?: { value: number; label: string } | null; + min?: number; + max?: number; + unit?: string; + /** Tooltip on the measure label (plain-English definition). */ + tip?: string; + /** Small note on the right of the header row (e.g. the tick legend). */ + headNote?: string; +} + +export function DotStrip({ + label, + values, + schoolNames, + anchor = null, + min = 0, + max = 100, + unit = '%', + tip, + headNote, +}: DotStripProps) { + const points = stripPositions(values, min, max); + const span = max - min; + const anchorPos = + anchor != null + ? Math.min(100, Math.max(0, ((anchor.value - min) / span) * 100)) + : null; + + const ariaParts = [ + anchor ? `${anchor.label}` : null, + ...points.map( + (p) => `${schoolNames[p.schoolIndex] ?? `School ${p.schoolIndex + 1}`} ${p.value}${unit}`, + ), + ].filter(Boolean); + + return ( +
+
+ + {label} + + {headNote && {headNote}} +
+
+
+ {anchorPos != null && anchor && ( + <> + + + {anchor.label} + + + )} + {points.map((p) => ( + + + + {p.value} + + + ))} +
+
+ ); +} From 9f2260ce5056eec706a8e315294c51626a506565 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:49:17 +0100 Subject: [PATCH 36/59] feat(compare): at-a-glance, Ofsted, admissions and community sections Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../components/CompareOfsted.test.tsx | 98 ++++++++ .../components/compare/CompareAdmissions.tsx | 124 ++++++++++ .../components/compare/CompareAtAGlance.tsx | 180 +++++++++++++++ .../components/compare/CompareCommunity.tsx | 183 +++++++++++++++ .../components/compare/CompareOfsted.tsx | 217 ++++++++++++++++++ .../compare/compareSections.module.css | 216 +++++++++++++++++ .../components/compare/sectionShared.tsx | 97 ++++++++ nextjs-app/lib/compareLogic.ts | 4 +- 8 files changed, 1117 insertions(+), 2 deletions(-) create mode 100644 nextjs-app/__tests__/components/CompareOfsted.test.tsx create mode 100644 nextjs-app/components/compare/CompareAdmissions.tsx create mode 100644 nextjs-app/components/compare/CompareAtAGlance.tsx create mode 100644 nextjs-app/components/compare/CompareCommunity.tsx create mode 100644 nextjs-app/components/compare/CompareOfsted.tsx create mode 100644 nextjs-app/components/compare/compareSections.module.css create mode 100644 nextjs-app/components/compare/sectionShared.tsx diff --git a/nextjs-app/__tests__/components/CompareOfsted.test.tsx b/nextjs-app/__tests__/components/CompareOfsted.test.tsx new file mode 100644 index 0000000..8df5602 --- /dev/null +++ b/nextjs-app/__tests__/components/CompareOfsted.test.tsx @@ -0,0 +1,98 @@ +import { render, screen } from '@testing-library/react'; + +import { CompareOfsted } from '@/components/compare/CompareOfsted'; +import type { ComparisonData, OfstedInspection, School } from '@/lib/types'; + +function school(urn: number, name: string): School { + return { urn, school_name: name } as School; +} + +function ofsted(partial: Partial): OfstedInspection { + return { + framework: null, + inspection_date: '2021-10-07', + inspection_type: null, + overall_effectiveness: null, + quality_of_education: null, + behaviour_attitudes: null, + personal_development: null, + leadership_management: null, + early_years_provision: null, + previous_overall: null, + rc_safeguarding_met: null, + rc_inclusion: null, + rc_curriculum_teaching: null, + rc_achievement: null, + rc_attendance_behaviour: null, + rc_personal_development: null, + rc_leadership_governance: null, + rc_early_years: null, + rc_sixth_form: null, + ofsted_page_url: 'https://reports.ofsted.gov.uk/provider/21/1', + ...partial, + }; +} + +const schools = [school(1, 'Graded School'), school(2, 'Carried School'), school(3, 'Card School')]; + +const data: Record = { + '1': { + school_info: schools[0], + yearly_data: [], + ofsted: ofsted({ overall_effectiveness: 1, grade_source: 'graded' }), + }, + '2': { + school_info: schools[1], + yearly_data: [], + ofsted: ofsted({ overall_effectiveness: 2, grade_source: 'ungraded_carried_forward' }), + }, + '3': { + school_info: schools[2], + yearly_data: [], + ofsted: ofsted({ + inspection_date: '2025-11-14', + rc_safeguarding_met: true, + report_card: { + rc_achievement: { code: 2, label: 'Strong standard' }, + rc_attendance_behaviour: { code: 4, label: 'Needs attention' }, + }, + }), + }, +}; + +describe('CompareOfsted', () => { + it('renders the three regimes without inventing an overall grade for report cards', () => { + render(); + + expect(screen.getByText('Outstanding')).toBeInTheDocument(); + // Carried-forward grade is shown but marked as such + expect(screen.getByText('Good')).toBeInTheDocument(); + expect(screen.getByText(/carried forward/i)).toBeInTheDocument(); + // Report card: label present, no overall-grade badge for that school + expect(screen.getByText('Report card')).toBeInTheDocument(); + expect(screen.getByText(/no overall grade/i)).toBeInTheDocument(); + }); + + it('uses one chip-list grammar for both regimes in judgement detail', () => { + render(); + // report-card area chip + expect(screen.getByText('Attendance & behaviour')).toBeInTheDocument(); + expect(screen.getByText('Needs attention')).toBeInTheDocument(); + // graded school without published subgrades → honest dataset statement + expect( + screen.getAllByText(/We don't hold area-by-area detail/i).length, + ).toBeGreaterThanOrEqual(1); + }); + + it('shows the mixed-regime comparability note only when regimes differ', () => { + render(); + expect(screen.getByText(/aren't directly comparable/i)).toBeInTheDocument(); + }); + + it('links every school to its Ofsted page', () => { + render(); + const links = screen.getAllByRole('link', { name: /Ofsted page/i }); + expect(links).toHaveLength(3); + expect(links[0]).toHaveAttribute('href', 'https://reports.ofsted.gov.uk/provider/21/1'); + }); +}); diff --git a/nextjs-app/components/compare/CompareAdmissions.tsx b/nextjs-app/components/compare/CompareAdmissions.tsx new file mode 100644 index 0000000..aeaa3dc --- /dev/null +++ b/nextjs-app/components/compare/CompareAdmissions.tsx @@ -0,0 +1,124 @@ +/** + * Getting a place — admissions framed the way the expert review requires: + * total applications are "named on N forms" (any preference rank, not + * head-to-head), one consistent chip metric (first-preference success), + * equal-preference and offers-vs-intake explanations up front. + */ + +'use client'; + +import { summariseAdmissions } from '@/lib/compareLogic'; +import type { ComparisonData, School } from '@/lib/types'; +import { CHART_COLORS } from '@/lib/utils'; +import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +export function CompareAdmissions({ + schools, + data, +}: { + schools: School[]; + data: Record; +}) { + const rows = schools.map((school) => data[String(school.urn)]?.admissions ?? null); + const anyData = rows.some(Boolean); + const entryYear = rows.find(Boolean)?.year; + const entryLabel = entryYear + ? `September ${String(entryYear).slice(0, 4)} entry` + : 'the most recent admissions round'; + + if (!anyData) { + return ( +
+ <> +
+ ); + } + + return ( +
+ From the most recent admissions round ({entryLabel}). "First choice" means + families who ranked the school top of their application form — officially a "first + preference". Schools never see your ranking: places are decided only by the + school's admission criteria, so listing a school lower down never hurts your chances. + These are National Offer Day offers — waiting lists and appeals can change the final + intake. + + } + > + + + Interest in the school + + {schools.map((school, i) => { + const a = rows[i]; + return ( + + {a?.total_applications != null && a?.places_offered != null ? ( + <> + Named on {a.total_applications.toLocaleString('en-GB')} forms ·{' '} + {a.places_offered.toLocaleString('en-GB')} places + + ) : ( + No data + )} + + ); + })} + + First-choice families offered a place + {schools.map((school, i) => { + const summary = summariseAdmissions(rows[i]); + return ( + + {summary.firstPrefPct != null ? ( + <> + {summary.firstPrefPct}%{' '} + {summary.chip && summary.chip.tone === 'warn' && ( + {summary.chip.text} + )} + + + + + ) : ( + No data + )} + + ); + })} + + What this means + {schools.map((school, i) => { + const a = rows[i]; + const summary = summariseAdmissions(a); + let text: string | null = null; + if (summary.firstPrefPct != null) { + if (summary.firstPrefPct >= 100) { + text = `Every family who put ${school.school_name} first got a place.`; + } else if (summary.firstPrefPct >= 90) { + text = `Nearly every family who put ${school.school_name} first got a place.`; + } else if (a?.oversubscribed) { + text = + 'More first-choice applications than places — check the school’s admission criteria (for most non-faith primaries, distance decides).'; + } else { + text = `${summary.firstPrefPct}% of first-choice families received an offer.`; + } + } + return ( + + {text ? {text} : —} + + ); + })} + +
+ ); +} diff --git a/nextjs-app/components/compare/CompareAtAGlance.tsx b/nextjs-app/components/compare/CompareAtAGlance.tsx new file mode 100644 index 0000000..a2f6605 --- /dev/null +++ b/nextjs-app/components/compare/CompareAtAGlance.tsx @@ -0,0 +1,180 @@ +/** + * At a glance — the short version of every section below it. Copy verbatim + * from the reviewed mockups. Report-card cells summarise by counting graded + * areas (best first) and always NAME problem areas; safeguarding is a + * separate line, never a count. + */ + +'use client'; + +import { + latestValues, + ofstedDisplay, + summariseAdmissions, + verdict, + type ReportCardSummary, +} from '@/lib/compareLogic'; +import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types'; +import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +function ReportCardChips({ summary }: { summary: ReportCardSummary }) { + return ( + <> + Report card + + {summary.counts.map((c) => ( + + {c.count} area{c.count === 1 ? '' : 's'} {c.label} + + ))} + {summary.problems.map((p) => ( + + {p.areaLabel}: {p.label} + + ))} + + + {summary.allClear && 'No areas need attention · '} + {summary.safeguarding === 'met' && 'Safeguarding met'} + {summary.safeguarding === 'not_met' && 'Safeguarding not met'} + + + ); +} + +export function CompareAtAGlance({ + schools, + data, + nationalAverages, + benchmarks, +}: { + schools: School[]; + data: Record; + nationalAverages?: NationalAverages; + benchmarks?: Benchmarks; +}) { + const urns = schools.map((school) => school.urn); + const isSecondary = schools.some( + (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, + ); + const headlineKey = isSecondary ? 'attainment_8_score' : 'rwm_expected_pct'; + const headlineValues = latestValues(data, urns, headlineKey); + const anchor = isSecondary + ? nationalAverages?.secondary?.attainment_8_score + : nationalAverages?.primary?.rwm_expected_pct; + const medianPupils = isSecondary + ? benchmarks?.secondary?.median_pupils + : benchmarks?.primary?.median_pupils; + + return ( +
+ + Latest Ofsted inspection + {schools.map((school, i) => { + const display = ofstedDisplay(data[String(school.urn)]?.ofsted); + return ( + + {display.kind === 'report_card' && } + {(display.kind === 'graded' || display.kind === 'carried_forward') && ( + <> + + {display.gradeLabel} + + {display.carriedForward && Grade carried forward} + + )} + {display.kind === 'none' && No inspection in our dataset} + + ); + })} + + + {isSecondary ? 'Attainment 8 score' : 'Children reaching the expected standard'} + + {schools.map((school, i) => { + const value = headlineValues[i]; + return ( + + {value != null ? ( + <> + {isSecondary ? value.toFixed(1) : `${Math.round(value)}%`}{' '} + {anchor != null && ( + + {verdict(value, anchor) === 'above' && 'Above England average'} + {verdict(value, anchor) === 'close' && 'Close to England average'} + {verdict(value, anchor) === 'below' && 'Below England average'} + + )} + {anchor != null && ( + + England average {isSecondary ? anchor.toFixed(1) : `${Math.round(anchor)}%`} + + )} + + ) : ( + No data + )} + + ); + })} + + Getting a place + {schools.map((school, i) => { + const summary = summariseAdmissions(data[String(school.urn)]?.admissions); + return ( + + {summary.chip ? ( + <> + {summary.chip.text} + {summary.interest && {summary.interest}} + + ) : ( + No admissions data + )} + + ); + })} + + Size + {schools.map((school, i) => { + const census = data[String(school.urn)]?.census; + const pupils = census?.total_pupils ?? school.total_pupils ?? null; + let sizeNote: string | null = null; + if (pupils != null && medianPupils != null) { + if (pupils >= medianPupils * 1.5) sizeNote = 'Much larger than average'; + else if (pupils >= medianPupils * 1.1) sizeNote = 'Larger than average'; + else if (pupils <= medianPupils * 0.66) sizeNote = 'Much smaller than average'; + else if (pupils <= medianPupils * 0.9) sizeNote = 'Smaller than average'; + else sizeNote = 'About average size'; + } + return ( + + {pupils != null ? ( + <> + {pupils.toLocaleString('en-GB')} pupils + {sizeNote && {sizeNote}} + + ) : ( + No data + )} + + ); + })} + +
+ ); +} diff --git a/nextjs-app/components/compare/CompareCommunity.tsx b/nextjs-app/components/compare/CompareCommunity.tsx new file mode 100644 index 0000000..9ae9cc3 --- /dev/null +++ b/nextjs-app/components/compare/CompareCommunity.tsx @@ -0,0 +1,183 @@ +/** + * Who goes there — the school's community from the latest census plus GIAS + * facts. Benchmark chips use the computed state-school averages and must + * carry their provenance wording (never "England average" for computed + * figures). Copy verbatim from the reviewed mockups. + */ + +'use client'; + +import { verdict } from '@/lib/compareLogic'; +import type { Benchmarks, ComparisonData, School } from '@/lib/types'; +import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +function pctSplit(part: number | null | undefined, total: number | null | undefined): string | null { + if (part == null || total == null || total === 0) return null; + return `${Math.round((part / total) * 100)}%`; +} + +export function CompareCommunity({ + schools, + data, + benchmarks, +}: { + schools: School[]; + data: Record; + benchmarks?: Benchmarks; +}) { + const isSecondary = schools.some( + (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, + ); + const bench = isSecondary ? benchmarks?.secondary : benchmarks?.primary; + + const fsmChip = (value: number | null) => { + if (value == null || bench?.disadvantaged_pct == null) return null; + const v = verdict(value, bench.disadvantaged_pct, 3); + return ( + + {v === 'above' && 'Above the state-school average'} + {v === 'close' && 'About the state-school average'} + {v === 'below' && 'Below the state-school average'} + + ); + }; + + return ( +
+ + Pupils on roll + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info as (School & { gias_total_pupils?: number | null; capacity?: number | null }) | undefined; + const census = data[String(school.urn)]?.census; + const pupils = census?.total_pupils ?? info?.gias_total_pupils ?? null; + const capacity = info?.capacity ?? null; + let capNote: string | null = null; + if (pupils != null && capacity != null && capacity > 0) { + capNote = + pupils >= capacity + ? `${capacity.toLocaleString('en-GB')} places — at or above capacity` + : `of ${capacity.toLocaleString('en-GB')} places (${Math.round((pupils / capacity) * 100)}% full)`; + } + return ( + + {pupils != null ? ( + <> + {pupils.toLocaleString('en-GB')} + {capNote && {capNote}} + + ) : ( + No data + )} + + ); + })} + + Girls / boys + {schools.map((school, i) => { + const census = data[String(school.urn)]?.census; + const girls = pctSplit(census?.female_pupils, census?.total_pupils); + const boys = pctSplit(census?.male_pupils, census?.total_pupils); + return ( + + {girls && boys ? `${girls} / ${boys}` : No data} + + ); + })} + + + Free school meals + + {schools.map((school, i) => { + const fsm = data[String(school.urn)]?.census?.fsm_pct ?? null; + return ( + + {fsm != null ? ( + <> + {Math.round(fsm)}% {fsmChip(fsm)} + + ) : ( + No data + )} + + ); + })} + + + English as an additional language + + {schools.map((school, i) => { + const eal = data[String(school.urn)]?.census?.eal_pct ?? null; + return ( + + {eal != null ? `${Math.round(eal)}%` : No data} + + ); + })} + + + Extra learning support (SEN) + + {schools.map((school, i) => { + const rows = data[String(school.urn)]?.yearly_data ?? []; + let sen: number | null = null; + for (let r = rows.length - 1; r >= 0; r--) { + if (rows[r].sen_support_pct != null) { + sen = rows[r].sen_support_pct; + break; + } + } + const high = + sen != null && bench?.sen_support_pct != null && sen >= bench.sen_support_pct * 1.75; + return ( + + {sen != null ? ( + <> + {Math.round(sen)}% {high && Well above average} + + ) : ( + No data + )} + + ); + })} + + Faith character + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info; + const faith = info?.religious_denomination; + const none = !faith || faith === 'Does not apply' || faith === 'None'; + return ( + + {none ? 'None' : faith} + + ); + })} + + Ages + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info; + return ( + + {info?.age_range || No data} + + ); + })} + + Run by + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info; + const trust = info?.trust_name; + const la = info?.local_authority ?? school.local_authority; + return ( + + {trust ? trust : la ? `${la} council` : No data} + + ); + })} + +
+ ); +} diff --git a/nextjs-app/components/compare/CompareOfsted.tsx b/nextjs-app/components/compare/CompareOfsted.tsx new file mode 100644 index 0000000..e148674 --- /dev/null +++ b/nextjs-app/components/compare/CompareOfsted.tsx @@ -0,0 +1,217 @@ +/** + * Ofsted section — one visual grammar for inspection detail across all + * three regimes (legacy graded, interim carried-forward, renewed-framework + * report card). Copy comes verbatim from the reviewed mockups. + */ + +'use client'; + +import { + OFSTED_LEGACY_GRADES, + ofstedDisplay, + rcAreaLabel, + type OfstedDisplay, +} from '@/lib/compareLogic'; +import type { ComparisonData, OfstedInspection, School } from '@/lib/types'; +import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +const GRADE_TONE: Record = { + 1: 'good', + 2: 'good', + 3: 'warn', + 4: 'bad', +}; + +const RC_CODE_TONE = (code: number): 'good' | 'warn' | 'bad' | 'neutral' => + code <= 2 ? 'good' : code === 3 ? 'neutral' : code === 4 ? 'warn' : 'bad'; + +function formatInspectionDate(iso: string | null): string { + if (!iso) return '—'; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return '—'; + return d.toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); +} + +function yearsSince(iso: string | null): number | null { + if (!iso) return null; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return null; + return (Date.now() - d.getTime()) / (365.25 * 24 * 3600 * 1000); +} + +function ResultCell({ display }: { display: OfstedDisplay }) { + if (display.kind === 'none') { + return No inspection outcome in our dataset; + } + if (display.kind === 'report_card') { + return ( + <> + Report card + New-style inspection — no overall grade is given + + ); + } + return ( + <> + + {display.gradeLabel} + + + {display.carriedForward + ? 'Grade carried forward from an earlier inspection (ungraded visit since)' + : 'Overall grade (older-style inspection)'} + + + ); +} + +function JudgementDetailCell({ + ofsted, + display, + schoolName, +}: { + ofsted: OfstedInspection; + display: OfstedDisplay; + schoolName: string; +}) { + if (display.kind === 'report_card') { + const entries = Object.entries(ofsted.report_card ?? {}); + return ( +
+ {entries.map(([key, entry]) => ( +
+ {rcAreaLabel(key)} + {entry.label} +
+ ))} + {ofsted.rc_safeguarding_met != null && ( +
+ Safeguarding + + {ofsted.rc_safeguarding_met ? 'Met' : 'Not met'} + +
+ )} +
+ ); + } + + const legacyAreas: Array<[string, number | null]> = [ + ['Quality of education', ofsted.quality_of_education], + ['Behaviour & attitudes', ofsted.behaviour_attitudes], + ['Personal development', ofsted.personal_development], + ['Leadership & management', ofsted.leadership_management], + ['Early years provision', ofsted.early_years_provision], + ]; + const published = legacyAreas.filter(([, grade]) => grade != null); + + if (published.length === 0) { + return ( + + We don't hold area-by-area detail for this inspection — see {schoolName}'s + Ofsted page for the full report. + + ); + } + return ( +
+ {published.map(([label, grade]) => ( +
+ {label} + + {OFSTED_LEGACY_GRADES[grade as number] ?? String(grade)} + +
+ ))} +
+ ); +} + +export function CompareOfsted({ + schools, + data, +}: { + schools: School[]; + data: Record; +}) { + const displays = schools.map((school) => ofstedDisplay(data[String(school.urn)]?.ofsted)); + const kinds = new Set(displays.map((d) => d.kind).filter((k) => k !== 'none')); + const mixedRegimes = kinds.size > 1; + + return ( +
+ Ofsted is the schools inspectorate. It stopped giving a single overall grade in{' '} + September 2024; inspections between then and November 2025 kept the + area-by-area judgements without an overall grade, and from November 2025{' '} + new inspections produce a report card rating each area of school life on + a five-point scale. + {mixedRegimes && ( + <> A report card and an older overall grade aren't directly comparable. + )}{' '} + (Ofsted's "Expected standard" rating is unrelated to the KS2 "expected + standard" test measure further down this page.) + + } + > + + Result + {schools.map((school, i) => ( + + + + ))} + + Inspected + {schools.map((school, i) => { + const ofsted = data[String(school.urn)]?.ofsted; + const age = yearsSince(ofsted?.inspection_date ?? null); + return ( + + {formatInspectionDate(ofsted?.inspection_date ?? null)}{' '} + {age != null && age > 4 && 4+ years ago} + + ); + })} + + + Judgement detail + + {schools.map((school, i) => { + const ofsted = data[String(school.urn)]?.ofsted; + return ( + + {ofsted ? ( + + ) : ( + No inspection in our dataset + )} + + ); + })} + + + Ofsted page + + {schools.map((school, i) => { + const url = + data[String(school.urn)]?.ofsted?.ofsted_page_url ?? + `https://reports.ofsted.gov.uk/provider/21/${school.urn}`; + return ( + + + {school.school_name}'s Ofsted page → + + + ); + })} + +
+ ); +} diff --git a/nextjs-app/components/compare/compareSections.module.css b/nextjs-app/components/compare/compareSections.module.css new file mode 100644 index 0000000..895adae --- /dev/null +++ b/nextjs-app/components/compare/compareSections.module.css @@ -0,0 +1,216 @@ +/* Shared layout for the compare screen's measure-first sections. + Mobile base: each row-label becomes a measure header and each school cell + stacks under it (colour-coded via the cell's ::before school tag). + Desktop (≥761px): the mockups' grid — 200px row-label column + one column + per school (2–4 columns supported via --school-count). */ + +.section { + margin-top: 3rem; +} + +.sectionTitle { + font-family: var(--font-playfair), 'Playfair Display', Georgia, serif; + font-size: 1.45rem; + font-weight: 700; + margin: 0; + padding-left: 0.75rem; + border-left: 3px solid var(--accent-coral-dark); +} + +.how { + font-size: 0.85rem; + color: var(--text-muted); + margin: 0.35rem 0 0 0.95rem; + max-width: 70ch; +} + +.grid { + display: grid; + grid-template-columns: 1fr; + gap: 0; + margin-top: 1.25rem; +} + +.rowLabel { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-secondary); + display: flex; + align-items: center; + gap: 0.35rem; + background: var(--bg-secondary); + border-radius: 6px; + padding: 0.4rem 0.6rem; + margin-top: 0.8rem; +} + +.cell { + padding: 0.4rem 0.6rem; + font-size: 0.95rem; +} + +.cell::before { + content: attr(data-school); + display: block; + font-size: 0.72rem; + font-weight: 600; + color: var(--sc, var(--text-muted)); +} + +.big { + font-size: 1.35rem; + font-weight: 700; + font-variant-numeric: tabular-nums; +} + +.small { + display: block; + font-size: 0.8rem; + color: var(--text-muted); + margin-top: 0.1rem; +} + +.chip { + display: inline-block; + font-size: 0.75rem; + font-weight: 600; + border-radius: 999px; + padding: 0.15rem 0.6rem; + white-space: nowrap; +} + +.chipGood { + background: rgba(45, 125, 125, 0.14); + color: var(--accent-teal); +} + +.chipWarn { + background: var(--accent-gold-bg); + color: var(--accent-gold-text); +} + +.chipBad { + background: var(--accent-coral-bg); + color: var(--accent-coral-dark); +} + +.chipNeutral { + background: var(--bg-secondary); + color: var(--text-secondary); +} + +.help { + display: inline-flex; + width: 15px; + height: 15px; + border-radius: 50%; + border: 1px solid var(--text-muted); + color: var(--text-muted); + font-size: 0.65rem; + align-items: center; + justify-content: center; + cursor: help; + flex: none; +} + +.badge { + display: inline-block; + font-weight: 700; + border-radius: 6px; + padding: 0.25rem 0.7rem; + font-size: 0.9rem; +} + +.badgeGood { + background: rgba(45, 125, 125, 0.14); + color: var(--accent-teal); +} + +.badgeWarn { + background: var(--accent-gold-bg); + color: var(--accent-gold-text); +} + +.badgeBad { + background: var(--accent-coral-bg); + color: var(--accent-coral-dark); +} + +.rcList { + display: flex; + flex-direction: column; + gap: 0.3rem; + margin-top: 0.2rem; +} + +.rcRow { + display: flex; + justify-content: space-between; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; +} + +.rcArea { + color: var(--text-secondary); +} + +.chipStack { + display: flex; + gap: 0.3rem; + flex-wrap: wrap; + margin-top: 0.3rem; +} + +.barMini { + display: block; + height: 8px; + border-radius: 4px; + background: var(--bg-secondary); + overflow: hidden; + margin-top: 0.3rem; + max-width: 140px; +} + +.barMini > i { + display: block; + height: 100%; + border-radius: 4px; +} + +.card { + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 16px; + box-shadow: var(--shadow-soft); + padding: 1.25rem 1.5rem; + margin-top: 1rem; +} + +.link { + color: var(--accent-coral-dark); +} + +@media (min-width: 761px) { + .grid { + grid-template-columns: 200px repeat(var(--school-count, 3), 1fr); + gap: 0 0.75rem; + } + + .rowLabel { + background: none; + border-radius: 0; + margin-top: 0; + padding: 0.85rem 0.5rem 0.85rem 0; + border-bottom: 1px solid var(--border-light); + } + + .cell { + padding: 0.85rem 0.25rem; + border-bottom: 1px solid var(--border-light); + } + + .cell::before { + content: none; + } +} diff --git a/nextjs-app/components/compare/sectionShared.tsx b/nextjs-app/components/compare/sectionShared.tsx new file mode 100644 index 0000000..89b5d7a --- /dev/null +++ b/nextjs-app/components/compare/sectionShared.tsx @@ -0,0 +1,97 @@ +/** + * Small shared pieces for the compare sections: the section shell, the + * row-label + per-school-cell grid, and tone-mapped chips. Copy passed into + * these comes verbatim from the reviewed mockups + * (docs/superpowers/specs/mockups/) — do not paraphrase it here. + */ + +'use client'; + +import type { CSSProperties, ReactNode } from 'react'; + +import type { School } from '@/lib/types'; +import { CHART_TEXT_COLORS } from '@/lib/utils'; +import styles from './compareSections.module.css'; + +export function Section({ + title, + how, + children, +}: { + title: string; + how?: ReactNode; + children: ReactNode; +}) { + return ( +
+

{title}

+ {how &&

{how}

} + {children} +
+ ); +} + +export function SectionGrid({ + schools, + children, +}: { + schools: School[]; + children: ReactNode; +}) { + return ( +
+ {children} +
+ ); +} + +export function RowLabel({ children, tip }: { children: ReactNode; tip?: string }) { + return ( +
+ {children} + {tip && ( + + ? + + )} +
+ ); +} + +export function Cell({ + school, + index, + children, +}: { + school: School; + index: number; + children: ReactNode; +}) { + return ( +
+ {children} +
+ ); +} + +export type ChipTone = 'good' | 'warn' | 'bad' | 'neutral'; + +const CHIP_TONE_CLASS: Record = { + good: styles.chipGood, + warn: styles.chipWarn, + bad: styles.chipBad, + neutral: styles.chipNeutral, +}; + +export function Chip({ tone, children }: { tone: ChipTone; children: ReactNode }) { + return {children}; +} + +export const sectionStyles = styles; diff --git a/nextjs-app/lib/compareLogic.ts b/nextjs-app/lib/compareLogic.ts index 3d4134f..0916161 100644 --- a/nextjs-app/lib/compareLogic.ts +++ b/nextjs-app/lib/compareLogic.ts @@ -229,14 +229,14 @@ export function stripPositions( /** Latest non-null yearly value of `metricKey` per school, in `urns` order. */ export function latestValues( - data: Record & { year: number }> }>, + data: Record }>, urns: number[], metricKey: string, ): Array { return urns.map((urn) => { const rows = data[String(urn)]?.yearly_data ?? []; for (let i = rows.length - 1; i >= 0; i--) { - const v = rows[i][metricKey]; + const v = (rows[i] as Record)[metricKey]; if (typeof v === 'number' && !Number.isNaN(v)) return v; } return null; From 2573cd2490e2361bdda01bec3de3c0f4d505fd03 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:50:29 +0100 Subject: [PATCH 37/59] feat(compare): academics strips with England anchors and More measures Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- nextjs-app/__tests__/lib/compareLogic.test.ts | 28 ++ .../compare/CompareAcademics.module.css | 18 ++ .../components/compare/CompareAcademics.tsx | 292 ++++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100644 nextjs-app/components/compare/CompareAcademics.module.css create mode 100644 nextjs-app/components/compare/CompareAcademics.tsx diff --git a/nextjs-app/__tests__/lib/compareLogic.test.ts b/nextjs-app/__tests__/lib/compareLogic.test.ts index d181a5d..03c2432 100644 --- a/nextjs-app/__tests__/lib/compareLogic.test.ts +++ b/nextjs-app/__tests__/lib/compareLogic.test.ts @@ -229,3 +229,31 @@ describe('stripPositions', () => { expect(pts[0].pos).toBe(0); }); }); + +describe('latestValues', () => { + const data = { + '1': { + yearly_data: [ + { year: 202324, rwm_expected_pct: 75 }, + { year: 202425, rwm_expected_pct: 87 }, + ], + }, + '2': { + yearly_data: [ + { year: 202324, rwm_expected_pct: 82 }, + { year: 202425, rwm_expected_pct: null }, + ], + }, + }; + + it('takes the latest non-null value per school in urn order', async () => { + const { latestValues } = await import('@/lib/compareLogic'); + expect(latestValues(data, [1, 2], 'rwm_expected_pct')).toEqual([87, 82]); + }); + + it('returns null for unknown schools and metrics', async () => { + const { latestValues } = await import('@/lib/compareLogic'); + expect(latestValues(data, [3], 'rwm_expected_pct')).toEqual([null]); + expect(latestValues(data, [1], 'nope')).toEqual([null]); + }); +}); diff --git a/nextjs-app/components/compare/CompareAcademics.module.css b/nextjs-app/components/compare/CompareAcademics.module.css new file mode 100644 index 0000000..42b36ef --- /dev/null +++ b/nextjs-app/components/compare/CompareAcademics.module.css @@ -0,0 +1,18 @@ +.moreMeasures { + margin-top: 0.5rem; + border-top: 1px solid var(--border-light); + padding-top: 0.75rem; +} + +.moreMeasures summary { + cursor: pointer; + font-weight: 600; + font-size: 0.88rem; + color: var(--accent-coral-dark); +} + +.stripNote { + font-size: 0.78rem; + color: var(--text-muted); + margin: 0.5rem 0 0; +} diff --git a/nextjs-app/components/compare/CompareAcademics.tsx b/nextjs-app/components/compare/CompareAcademics.tsx new file mode 100644 index 0000000..d31c348 --- /dev/null +++ b/nextjs-app/components/compare/CompareAcademics.tsx @@ -0,0 +1,292 @@ +/** + * How children do academically — tier-1 dot strips anchored on official + * England averages, tier-2 "More measures" one tap away, equity row against + * the computed state-school benchmark. Copy verbatim from the reviewed + * mockups; teacher-assessed measures are labelled as such. + */ + +'use client'; + +import { latestValues, verdict } from '@/lib/compareLogic'; +import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types'; +import { DotStrip } from '@/components/DotStrip'; +import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; +import styles from './CompareAcademics.module.css'; + +interface StripSpec { + label: string; + metric: string; + anchorKey?: string; + tip?: string; + min?: number; + max?: number; + unit?: string; +} + +const TIER1_PRIMARY: StripSpec[] = [ + { + label: 'Reading, writing & maths — expected standard', + metric: 'rwm_expected_pct', + anchorKey: 'rwm_expected_pct', + tip: '% of Year 6 pupils reaching the expected standard in reading, writing and maths.', + }, + { label: 'Reading', metric: 'reading_expected_pct', anchorKey: 'reading_expected_pct' }, + { + label: 'Writing (teacher-assessed)', + metric: 'writing_expected_pct', + anchorKey: 'writing_expected_pct', + tip: 'Writing is assessed by teachers, not tested.', + }, + { label: 'Maths', metric: 'maths_expected_pct', anchorKey: 'maths_expected_pct' }, + { + label: 'Working at a higher standard than expected', + metric: 'rwm_high_pct', + anchorKey: 'rwm_high_pct', + tip: 'A high score in the reading and maths tests plus “greater depth” in teacher-assessed writing.', + }, +]; + +const TIER2_PRIMARY: StripSpec[] = [ + { + label: 'Grammar, punctuation & spelling — expected standard', + metric: 'gps_expected_pct', + anchorKey: 'gps_expected_pct', + }, + { + label: 'Science — expected standard (teacher-assessed)', + metric: 'science_expected_pct', + anchorKey: 'science_expected_pct', + tip: 'Teacher-assessed, like writing — there has been no KS2 science test since 2009, so comparisons are indicative.', + }, + { + label: 'Average scaled score — reading', + metric: 'reading_avg_score', + anchorKey: 'reading_avg_score', + min: 100, + max: 120, + unit: '', + }, + { + label: 'Average scaled score — maths', + metric: 'maths_avg_score', + anchorKey: 'maths_avg_score', + min: 100, + max: 120, + unit: '', + }, + { + label: 'Average scaled score — grammar, punctuation & spelling', + metric: 'gps_avg_score', + anchorKey: 'gps_avg_score', + min: 100, + max: 120, + unit: '', + }, +]; + +function Strip({ + spec, + data, + urns, + schoolNames, + national, +}: { + spec: StripSpec; + data: Record; + urns: number[]; + schoolNames: string[]; + national: Record | undefined; +}) { + const values = latestValues(data, urns, spec.metric).map((v) => + v != null ? Math.round(v) : null, + ); + const anchorValue = spec.anchorKey ? national?.[spec.anchorKey] : undefined; + const anchor = + anchorValue != null + ? { value: anchorValue, label: `England ${Math.round(anchorValue)}${spec.unit ?? '%'}` } + : null; + if (values.every((v) => v == null)) return null; + return ( + + ); +} + +export function CompareAcademics({ + schools, + data, + nationalAverages, + benchmarks, +}: { + schools: School[]; + data: Record; + nationalAverages?: NationalAverages; + benchmarks?: Benchmarks; +}) { + const urns = schools.map((school) => school.urn); + const schoolNames = schools.map((school) => school.school_name); + const isSecondary = schools.some( + (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, + ); + + if (isSecondary) { + const att8 = latestValues(data, urns, 'attainment_8_score'); + const banding = urns.map((urn) => { + const rows = data[String(urn)]?.yearly_data ?? []; + for (let i = rows.length - 1; i >= 0; i--) { + if (rows[i].progress_8_banding) return rows[i].progress_8_banding as string; + } + return null; + }); + const grade5 = latestValues(data, urns, 'english_maths_strong_pass_pct'); + const ebacc = latestValues(data, urns, 'ebacc_entry_pct'); + const att8Anchor = nationalAverages?.secondary?.attainment_8_score; + + return ( +
+ + Attainment 8 + {schools.map((school, i) => ( + + {att8[i] != null ? ( + <> + {(att8[i] as number).toFixed(1)} + {att8Anchor != null && ( + England average {att8Anchor.toFixed(1)} + )} + + ) : ( + No data + )} + + ))} + + Progress 8 + {schools.map((school, i) => ( + + {banding[i] ? ( + + {banding[i]} + + ) : ( + No data + )} + + ))} + + + Grade 5+ in English & maths + + {schools.map((school, i) => ( + + {grade5[i] != null ? `${Math.round(grade5[i] as number)}%` : No data} + + ))} + + EBacc entry + {schools.map((school, i) => ( + + {ebacc[i] != null ? `${Math.round(ebacc[i] as number)}%` : No data} + + ))} + +
+ ); + } + + const national = nationalAverages?.primary; + const disadvantaged = latestValues(data, urns, 'rwm_expected_disadvantaged_pct'); + const disadvantagedAnchor = benchmarks?.primary?.disadvantaged_rwm_expected_pct ?? null; + + return ( +
+
+ {TIER1_PRIMARY.map((spec) => ( + + ))} + +
+ More measures — grammar, punctuation & spelling, science, average scaled scores + {TIER2_PRIMARY.map((spec) => ( + + ))} +

+ The scaled-score strips show the 100–120 window of the full 80–120 range; 100 is the + expected standard. Where an England tick is missing, the official figure isn't in + our dataset yet. +

+
+
+ + {disadvantaged.some((v) => v != null) && ( + + + Children from lower-income families + + {schools.map((school, i) => { + const value = disadvantaged[i]; + return ( + + {value != null ? ( + <> + + {Math.round(value)}% + {' '} + {disadvantagedAnchor != null && ( + + {verdict(value, disadvantagedAnchor, 5) === 'above' && + `Well above the ${Math.round(disadvantagedAnchor)}% state-school average`} + {verdict(value, disadvantagedAnchor, 5) === 'close' && + `Around the ${Math.round(disadvantagedAnchor)}% state-school average`} + {verdict(value, disadvantagedAnchor, 5) === 'below' && + `Below the ${Math.round(disadvantagedAnchor)}% state-school average`} + + )} + + ) : ( + No data + )} + + ); + })} + + )} +
+ ); +} From 519584f34ba8d56d0bad9566e3799e1b26b3de88 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:54:20 +0100 Subject: [PATCH 38/59] feat(compare): trends explorer with England line; gap-honest axis; series regression guard Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../__tests__/lib/compareChartData.test.ts | 84 ++++++++ .../components/ComparisonChart.module.css | 6 + nextjs-app/components/ComparisonChart.tsx | 56 +++-- .../compare/TrendsExplorer.module.css | 96 +++++++++ .../components/compare/TrendsExplorer.tsx | 195 ++++++++++++++++++ nextjs-app/lib/compareChartData.ts | 102 +++++++++ 6 files changed, 523 insertions(+), 16 deletions(-) create mode 100644 nextjs-app/__tests__/lib/compareChartData.test.ts create mode 100644 nextjs-app/components/compare/TrendsExplorer.module.css create mode 100644 nextjs-app/components/compare/TrendsExplorer.tsx create mode 100644 nextjs-app/lib/compareChartData.ts diff --git a/nextjs-app/__tests__/lib/compareChartData.test.ts b/nextjs-app/__tests__/lib/compareChartData.test.ts new file mode 100644 index 0000000..09325ae --- /dev/null +++ b/nextjs-app/__tests__/lib/compareChartData.test.ts @@ -0,0 +1,84 @@ +/** + * buildCompareChart: every selected school must produce a rendered series + * (regression guard for the production bug where a third school's line + * vanished), the x-axis must include cancelled/unpublished years as real + * gaps (never compressing time), and the England overlay renders dashed + * with no gap-bridging. + */ + +import { buildCompareChart, fillAcademicYears } from '@/lib/compareChartData'; +import type { ComparisonData } from '@/lib/types'; + +function school(urn: number, years: Array<[number, number | null]>): ComparisonData { + return { + school_info: { urn, school_name: `School ${urn}` } as ComparisonData['school_info'], + yearly_data: years.map(([year, v]) => ({ year, rwm_expected_pct: v })) as ComparisonData['yearly_data'], + }; +} + +const THREE_SCHOOLS = { + '1': school(1, [[201819, 87], [202223, 87], [202425, 87]]), + '2': school(2, [[201819, 88], [202223, 88], [202425, 92]]), + '3': school(3, [[201819, 69], [202223, 62], [202425, 79]]), +}; + +const SCHOOL_LIST = [1, 2, 3].map((urn) => ({ urn, school_name: `School ${urn}` })); + +describe('fillAcademicYears', () => { + it('fills every academic year between min and max', () => { + expect(fillAcademicYears([201819, 202223])).toEqual([ + 201819, 201920, 202021, 202122, 202223, + ]); + }); +}); + +describe('buildCompareChart', () => { + it('renders one series per selected school — none silently dropped', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(chart.schoolDatasets).toHaveLength(3); + for (const ds of chart.schoolDatasets) { + expect(ds.data.some((v) => v != null)).toBe(true); + } + }); + + it('handles float years from the API (202425.0 style)', () => { + const floaty = { + '1': school(1, [[201819.0 as number, 80], [202425.0 as number, 85]]), + }; + const chart = buildCompareChart(floaty, [SCHOOL_LIST[0]], 'rwm_expected_pct'); + expect(chart.schoolDatasets[0].data.filter((v) => v != null)).toHaveLength(2); + }); + + it('includes cancelled/unpublished years as null gaps, not compressed time', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(chart.years).toContain(201920); + expect(chart.years).toContain(202122); + const idx = chart.years.indexOf(202021); + expect(chart.schoolDatasets[0].data[idx]).toBeNull(); + }); + + it('adds a dashed England overlay when national data is supplied', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', { + 201819: 64.9, + 202122: 58.7, + 202223: 59.5, + 202425: 62.1, + }); + expect(chart.englandDataset).not.toBeNull(); + const eng = chart.englandDataset!; + expect(eng.label).toBe('England average'); + expect(eng.borderDash).toEqual([5, 4]); + expect(eng.spanGaps).toBe(false); + // England has a value for 2021/22 even though schools do not + expect(eng.data[chart.years.indexOf(202122)]).toBe(58.7); + }); + + it('flags the unpublished 2021/22 school-level year when England has data but schools do not', () => { + const withNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', { + 202122: 58.7, + }); + expect(withNational.showUnpublished202122Note).toBe(true); + const withoutNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(withoutNational.showUnpublished202122Note).toBe(false); + }); +}); diff --git a/nextjs-app/components/ComparisonChart.module.css b/nextjs-app/components/ComparisonChart.module.css index dbd82c6..d847f1d 100644 --- a/nextjs-app/components/ComparisonChart.module.css +++ b/nextjs-app/components/ComparisonChart.module.css @@ -65,3 +65,9 @@ min-width: 0; } } + +.chartNote { + font-size: 0.78rem; + color: var(--text-muted); + margin: 0.5rem 0 0; +} diff --git a/nextjs-app/components/ComparisonChart.tsx b/nextjs-app/components/ComparisonChart.tsx index d922991..851b03d 100644 --- a/nextjs-app/components/ComparisonChart.tsx +++ b/nextjs-app/components/ComparisonChart.tsx @@ -15,6 +15,7 @@ import { useEffect, useState } from 'react'; import { Line } from 'react-chartjs-2'; import { ChartOptions, ChartDataset, PointStyle } from 'chart.js'; import '@/lib/chartSetup'; +import { buildCompareChart } from '@/lib/compareChartData'; import type { ComparisonData } from '@/lib/types'; import { CHART_COLORS, @@ -34,13 +35,16 @@ interface ComparisonChartProps { schools: Array<{ urn: number; school_name: string }>; metric: string; metricLabel: string; + /** Official England figure per academic year for this metric — renders a + * dashed grey reference line when provided. */ + nationalByYear?: Record; } // One shape per basket slot (MAX_SCHOOLS = 5) — secondary encoding so // converging lines stay tellable apart without relying on hue alone. const POINT_STYLES: PointStyle[] = ['circle', 'triangle', 'rect', 'rectRot', 'star']; -export function ComparisonChart({ comparisonData, schools, metric, metricLabel }: ComparisonChartProps) { +export function ComparisonChart({ comparisonData, schools, metric, metricLabel, nationalByYear }: ComparisonChartProps) { const isMobile = useIsMobile(); const [focusedUrn, setFocusedUrn] = useState(null); @@ -54,34 +58,48 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel } return
No data available
; } - // Union of years across all schools — coverage differs between them. - const years = [ - ...new Set(schools.flatMap((s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => d.year) ?? [])), - ].sort((a, b) => a - b); + // Pure, tested series construction: union of years with cancelled / + // unpublished years kept as real gaps, plus the England overlay. + const built = buildCompareChart(comparisonData, schools, metric, nationalByYear); + const { years } = built; - const datasets: ChartDataset<'line'>[] = schools.map((school, index) => { - const data = comparisonData[String(school.urn)]; - const color = CHART_COLORS[index % CHART_COLORS.length]; + const datasets: ChartDataset<'line'>[] = built.schoolDatasets.map((series) => { + const school = schools[series.schoolIndex]; + const color = CHART_COLORS[series.schoolIndex % CHART_COLORS.length]; const dimmed = focusedUrn !== null && focusedUrn !== school.urn; return { - label: school.school_name, - data: years.map((year) => { - const yearData = data?.yearly_data.find((d) => d.year === year); - if (!yearData) return null; - return yearData[metric as keyof typeof yearData] as number | null; - }), + label: series.label, + data: series.data, borderColor: dimmed ? rgbToRgba(color, 0.2) : color, backgroundColor: dimmed ? 'transparent' : rgbToRgba(color, 0.1), borderWidth: focusedUrn === school.urn ? 3 : dimmed ? 1.5 : 2, - pointStyle: POINT_STYLES[index % POINT_STYLES.length], + pointStyle: POINT_STYLES[series.schoolIndex % POINT_STYLES.length], pointRadius: dimmed ? 2 : isMobile ? 3 : 4, pointHoverRadius: isMobile ? 5 : 6, tension: 0.3, - spanGaps: true, + // Never bridge missing years — gaps are information (COVID + // cancellations, unpublished 2021/22, schools that opened later). + spanGaps: false, }; }); + if (built.englandDataset) { + datasets.push({ + label: built.englandDataset.label, + data: built.englandDataset.data, + borderColor: 'rgba(109, 104, 95, 0.9)', + backgroundColor: 'transparent', + borderWidth: 1.5, + borderDash: built.englandDataset.borderDash, + pointStyle: 'line', + pointRadius: 0, + pointHoverRadius: 4, + tension: 0, + spanGaps: false, + }); + } + const chartData = { labels: years.map(formatAcademicYear), datasets, @@ -222,6 +240,12 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
+ {built.showUnpublished202122Note && ( +

+ No national tests were held in 2019/20 and 2020/21 (COVID), and DfE didn't publish + school-level figures for 2021/22 — the England average is shown for that year. +

+ )}
); } diff --git a/nextjs-app/components/compare/TrendsExplorer.module.css b/nextjs-app/components/compare/TrendsExplorer.module.css new file mode 100644 index 0000000..d6ddd4d --- /dev/null +++ b/nextjs-app/components/compare/TrendsExplorer.module.css @@ -0,0 +1,96 @@ +.explore { + margin-top: 1rem; +} + +.explore summary { + cursor: pointer; + font-weight: 600; + color: var(--accent-coral-dark); + padding: 0.85rem 1.1rem; + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 8px; +} + +.explore[open] summary { + border-radius: 8px 8px 0 0; +} + +.inner { + border: 1px solid var(--border-light); + border-top: none; + border-radius: 0 0 8px 8px; + background: var(--bg-card); + padding: 1.25rem 1.5rem; +} + +.picker { + display: flex; + align-items: center; + gap: 0.6rem; + margin-bottom: 1rem; + flex-wrap: wrap; +} + +.picker label { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-secondary); +} + +.picker select { + font-family: inherit; + font-size: 0.9rem; + padding: 0.4rem 0.6rem; + border-radius: 8px; + border: 1px solid var(--border-light); + background: var(--bg-card); + color: var(--text-primary); + max-width: 100%; +} + +.desc { + font-size: 0.78rem; + color: var(--text-muted); +} + +.progressNote { + font-size: 0.8rem; + color: var(--text-muted); + margin: 0 0 1rem; +} + +.chartBox { + min-height: 320px; +} + +.tableWrapper { + overflow-x: auto; + margin-top: 1.5rem; +} + +.table { + width: 100%; + border-collapse: collapse; + font-size: 0.9rem; +} + +.table th, +.table td { + text-align: left; + padding: 0.6rem 0.75rem; + border-bottom: 1px solid var(--border-light); +} + +.table th { + background: var(--bg-secondary); + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.03em; + color: var(--text-secondary); +} + +.yearCell { + font-weight: 600; + white-space: nowrap; +} diff --git a/nextjs-app/components/compare/TrendsExplorer.tsx b/nextjs-app/components/compare/TrendsExplorer.tsx new file mode 100644 index 0000000..c4ffd64 --- /dev/null +++ b/nextjs-app/components/compare/TrendsExplorer.tsx @@ -0,0 +1,195 @@ +/** + * Explore trends — the full grouped metric catalogue (nothing from the old + * compare page is lost; spec §4's tier 3) driving the year-by-year chart + * with its England reference line, plus the year-by-year table. Progress + * metrics carry CI-based bands for the years DfE published them. + */ + +'use client'; + +import { useState } from 'react'; +import dynamic from 'next/dynamic'; + +import { progressBand } from '@/lib/compareLogic'; +import type { ComparisonData, MetricDefinition, NationalAverages, School } from '@/lib/types'; +import { formatAcademicYear, formatMetricValue, metricKind } from '@/lib/utils'; +import { track } from '@/lib/analytics'; +import { Chip, Section, sectionStyles as s } from './sectionShared'; +import styles from './TrendsExplorer.module.css'; + +const ComparisonChart = dynamic( + () => import('../ComparisonChart').then((m) => m.ComparisonChart), + { ssr: false }, +); + +const PRIMARY_OPTGROUPS: { label: string; category: string }[] = [ + { label: 'Expected Standard', category: 'expected' }, + { label: 'Higher Standard', category: 'higher' }, + { label: 'Progress Scores', category: 'progress' }, + { label: 'Average Scores', category: 'average' }, + { label: 'Gender Performance', category: 'gender' }, + { label: 'Equity (Disadvantaged)', category: 'equity' }, + { label: 'School Context', category: 'context' }, + { label: 'Absence', category: 'absence' }, + { label: '3-Year Trends', category: 'trends' }, +]; + +const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [ + { label: 'GCSE Performance', category: 'gcse' }, +]; + +export const PRIMARY_CATEGORIES = PRIMARY_OPTGROUPS.map((g) => g.category); +export const SECONDARY_CATEGORIES = SECONDARY_OPTGROUPS.map((g) => g.category); + +const PROGRESS_CI: Record = { + reading_progress: ['reading_progress_lower_ci', 'reading_progress_upper_ci'], + writing_progress: ['writing_progress_lower_ci', 'writing_progress_upper_ci'], + maths_progress: ['maths_progress_lower_ci', 'maths_progress_upper_ci'], +}; + +const BAND_LABEL = { above: 'Above average', average: 'Average', below: 'Below average' } as const; + +export function TrendsExplorer({ + schools, + data, + metrics, + initialMetric, + isPrimaryPhase, + nationalAverages, +}: { + schools: School[]; + data: Record; + metrics: MetricDefinition[]; + initialMetric: string; + isPrimaryPhase: boolean; + nationalAverages?: NationalAverages; +}) { + const [metric, setMetric] = useState(initialMetric); + + const allowedCategories = isPrimaryPhase ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES; + const optgroups = isPrimaryPhase ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS; + const filteredMetrics = metrics.filter((m) => allowedCategories.includes(m.category)); + const metricDef = metrics.find((m) => m.key === metric); + const metricLabel = metricDef?.label || metric; + + const nationalByYear: Record = {}; + for (const entry of nationalAverages?.by_year ?? []) { + const block = isPrimaryPhase ? entry.primary : entry.secondary; + nationalByYear[entry.year] = block?.[metric] ?? null; + } + + const years = [ + ...new Set( + schools.flatMap( + (school) => data[String(school.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [], + ), + ), + ].sort((a, b) => a - b); + + const handleMetricChange = (next: string) => { + track('compare_metric_changed', { metric: next, phase: isPrimaryPhase ? 'primary' : 'secondary' }); + setMetric(next); + }; + + const ciKeys = PROGRESS_CI[metric]; + + return ( +
+
+ Year-by-year trends +
+
+ + + {metricDef?.description && {metricDef.description}} +
+ + {metric.includes('progress') && ( +

+ Progress scores measure pupils' progress from KS1 to KS2. A score of 0 equals the + national average. DfE stopped publishing KS2 progress after 2022/23 (no KS1 baseline); + bands use DfE's confidence intervals, not the raw score alone. +

+ )} + +
+ +
+ + {years.length > 0 && ( +
+ + + + + {schools.map((school) => ( + + ))} + + + + {years.map((year) => ( + + + {schools.map((school) => { + const row = data[String(school.urn)]?.yearly_data.find( + (d) => Math.trunc(d.year) === year, + ) as (Record & { year: number }) | undefined; + const value = row?.[metric]; + if (typeof value !== 'number') return ; + const band = ciKeys + ? progressBand( + value, + (row?.[ciKeys[0]] as number | null) ?? null, + (row?.[ciKeys[1]] as number | null) ?? null, + ) + : null; + return ( + + ); + })} + + ))} + +
Year{school.school_name}
{formatAcademicYear(year)}– + {formatMetricValue(value, metricKind(metric))}{' '} + {band && ( + + {BAND_LABEL[band]} + + )} +
+
+ )} +
+
+
+ ); +} diff --git a/nextjs-app/lib/compareChartData.ts b/nextjs-app/lib/compareChartData.ts new file mode 100644 index 0000000..728fd7a --- /dev/null +++ b/nextjs-app/lib/compareChartData.ts @@ -0,0 +1,102 @@ +/** + * Pure series-building for the comparison trend chart, extracted from + * ComparisonChart so it is unit-testable without a canvas. + * + * Chart truthfulness rules (spec §8.1): every academic year between the + * first and last data point appears on the axis — cancelled test years + * (2019/20, 2020/21) and the unpublished 2021/22 school-level year render + * as real gaps, never as compressed time; school lines never bridge gaps. + */ + +import type { ComparisonData } from './types'; + +/** 201819 → 201920 (academic-year arithmetic on YYYYYY codes). */ +function nextAcademicYear(year: number): number { + const start = Math.floor(year / 100); + const end = year % 100; + return (start + 1) * 100 + (end + 1); +} + +/** Every academic year from min(years) to max(years), inclusive. */ +export function fillAcademicYears(years: number[]): number[] { + if (years.length === 0) return []; + const ints = [...new Set(years.map((y) => Math.trunc(y)))].sort((a, b) => a - b); + const out: number[] = []; + let y = ints[0]; + const last = ints[ints.length - 1]; + while (y <= last && out.length < 50) { + out.push(y); + y = nextAcademicYear(y); + } + return out; +} + +export interface CompareChartSeries { + label: string; + data: Array; + /** Index into CHART_COLORS / point styles. */ + schoolIndex: number; + spanGaps: false; +} + +export interface EnglandSeries { + label: 'England average'; + data: Array; + borderDash: [number, number]; + spanGaps: false; +} + +export interface CompareChart { + years: number[]; + schoolDatasets: CompareChartSeries[]; + englandDataset: EnglandSeries | null; + /** True when England published a 2021/22 figure but no school has one — + * the UI shows: "DfE didn't publish school-level figures for 2021/22". */ + showUnpublished202122Note: boolean; +} + +export function buildCompareChart( + comparisonData: Record, + schools: Array<{ urn: number; school_name: string }>, + metric: string, + nationalByYear?: Record, +): CompareChart { + const rawYears = schools.flatMap( + (s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [], + ); + const years = fillAcademicYears(rawYears); + + const schoolDatasets: CompareChartSeries[] = schools.map((school, schoolIndex) => { + const rows = comparisonData[String(school.urn)]?.yearly_data ?? []; + const byYear = new Map>(); + for (const row of rows) byYear.set(Math.trunc(row.year), row as unknown as Record); + return { + label: school.school_name, + data: years.map((year) => { + const v = byYear.get(year)?.[metric]; + return typeof v === 'number' && !Number.isNaN(v) ? v : null; + }), + schoolIndex, + spanGaps: false, + }; + }); + + let englandDataset: EnglandSeries | null = null; + if (nationalByYear) { + const data = years.map((year) => { + const v = nationalByYear[year]; + return typeof v === 'number' && !Number.isNaN(v) ? v : null; + }); + if (data.some((v) => v != null)) { + englandDataset = { label: 'England average', data, borderDash: [5, 4], spanGaps: false }; + } + } + + const idx202122 = years.indexOf(202122); + const showUnpublished202122Note = + idx202122 >= 0 && + englandDataset?.data[idx202122] != null && + schoolDatasets.every((ds) => ds.data[idx202122] == null); + + return { years, schoolDatasets, englandDataset, showUnpublished202122Note }; +} From 2155256177e52021cd10a9ec44ef2dda52e93374 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:58:00 +0100 Subject: [PATCH 39/59] feat(compare): parent-first compare screen assembly Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- nextjs-app/app/compare/page.tsx | 6 +- .../components/ComparisonView.module.css | 456 +++--------------- nextjs-app/components/ComparisonView.tsx | 429 ++++++---------- .../components/compare/TrendsExplorer.tsx | 12 +- 4 files changed, 240 insertions(+), 663 deletions(-) diff --git a/nextjs-app/app/compare/page.tsx b/nextjs-app/app/compare/page.tsx index 1a1ff38..31689a5 100644 --- a/nextjs-app/app/compare/page.tsx +++ b/nextjs-app/app/compare/page.tsx @@ -16,8 +16,10 @@ interface ComparePageProps { export const metadata: Metadata = { title: 'Compare Schools', - description: 'Compare KS2 performance across multiple primary schools in England', - keywords: 'school comparison, compare schools, KS2 comparison, primary school performance', + description: + 'Compare schools in England side by side — Ofsted inspections, KS2 and GCSE results against the England average, admissions odds and school community.', + keywords: + 'school comparison, compare schools, Ofsted comparison, school admissions, KS2 comparison, primary school performance', }; // Dynamic via searchParams; remove force-dynamic so internal data fetches diff --git a/nextjs-app/components/ComparisonView.module.css b/nextjs-app/components/ComparisonView.module.css index b6f1af6..633ba12 100644 --- a/nextjs-app/components/ComparisonView.module.css +++ b/nextjs-app/components/ComparisonView.module.css @@ -28,8 +28,15 @@ color: var(--text-secondary, #5c564d); margin: 0; line-height: 1.6; + max-width: 60ch; } +.headerActions { + display: flex; + gap: 0.75rem; + align-items: center; + flex-wrap: wrap; +} /* Phase Tabs */ .phaseTabs { @@ -72,408 +79,87 @@ background: var(--accent-coral-darker, #9c3f26); } -/* Metric Selector */ -.metricSelector { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; - padding: 1.5rem; - margin-bottom: 2rem; +/* Sticky school bar — column identity while scrolling; horizontal scroll on + narrow screens */ +.schoolBar { + position: sticky; + top: 0; + z-index: 10; + background: var(--bg-primary, #faf7f2); display: flex; - align-items: center; - flex-wrap: wrap; - gap: 1rem; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); -} - -.metricLabel { - font-size: 0.9375rem; - font-weight: 600; - color: var(--text-primary, #1a1612); - white-space: nowrap; -} - -.metricSelect { - flex: 1; - max-width: 400px; - padding: 0.625rem 1rem; - font-size: 0.9375rem; - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 8px; - background: var(--bg-card, white); - color: var(--text-primary, #1a1612); - cursor: pointer; - transition: all 0.2s ease; -} - -.metricSelect:hover { - border-color: var(--accent-coral, #e07256); -} - -.metricSelect:focus { - outline: none; - border-color: var(--accent-coral, #e07256); - box-shadow: 0 0 0 3px var(--accent-coral-bg); -} - -.metricSelect optgroup { - font-weight: 700; - color: var(--text-primary, #1a1612); - background: var(--bg-secondary, #f3ede4); - padding: 0.5rem 0; -} - -.metricSelect option { - font-weight: 400; - color: var(--text-secondary, #5c564d); - padding: 0.375rem 1rem; -} - -/* Schools Section */ -.schoolsSection { - margin-bottom: 2rem; -} - -.schoolsGrid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); - gap: 1.5rem; -} - -.schoolCard { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-left: 3px solid var(--accent-teal, #2d7d7d); - border-radius: 12px; - padding: 1.5rem; - position: relative; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); - transition: all 0.3s ease; - display: flex; - flex-direction: column; -} - -.schoolCard:hover { - box-shadow: var(--shadow-medium, 0 4px 20px rgba(26, 22, 18, 0.1)); - transform: translateY(-2px); -} - -.removeButton { - position: absolute; - top: 0.75rem; - right: 0.75rem; - width: 28px; - height: 28px; - display: flex; - align-items: center; - justify-content: center; - background: var(--accent-coral, #e07256); - color: white; - border: none; - border-radius: 50%; - font-size: 1.25rem; - line-height: 1; - cursor: pointer; - transition: all 0.2s ease; -} - -.removeButton:hover { - background: var(--accent-coral-dark, #c45a3f); - transform: scale(1.1); -} - -.schoolName { - font-size: 1.125rem; - font-weight: 600; - margin-bottom: 0.75rem; - padding-right: 2rem; - line-height: 1.3; - font-family: var(--font-playfair), 'Playfair Display', serif; -} - -.schoolName a { - color: var(--text-primary, #1a1612); - text-decoration: none; - transition: color 0.2s ease; -} - -.schoolName a:hover { - color: var(--accent-coral-dark, #b04a2e); -} - -.schoolMeta { - display: flex; - flex-direction: column; - gap: 0.5rem; - margin-bottom: 1rem; - flex: 1; -} - -.metaItem { - font-size: 0.875rem; - color: var(--text-secondary, #5c564d); - display: flex; - align-items: center; - gap: 0.25rem; -} - -.latestValue { - margin-top: auto; - padding-top: 1rem; - border-top: 1px solid var(--border-color, #e5dfd5); - text-align: center; - background: var(--bg-secondary, #f3ede4); - margin-left: -1.5rem; - margin-right: -1.5rem; - margin-bottom: -1.5rem; - padding: 1.25rem 1.5rem; - border-radius: 0 0 12px 9px; -} - -.latestLabel { - font-size: 0.75rem; - color: var(--text-muted, #8a847a); - margin-bottom: 0.25rem; - text-transform: uppercase; - letter-spacing: 0.05em; -} - -.latestNumber { - font-size: 1.75rem; - font-weight: 700; - color: var(--accent-teal, #2d7d7d); -} - -/* Chart Section */ -.chartSection { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; - padding: 2rem; - margin-bottom: 2rem; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); -} - -.sectionTitle { - font-size: 1.5rem; - font-weight: 600; - color: var(--text-primary, #1a1612); - margin-bottom: 1.5rem; - padding-bottom: 0.75rem; - border-bottom: 2px solid var(--border-color, #e5dfd5); - font-family: var(--font-playfair), 'Playfair Display', serif; - display: flex; - align-items: center; - gap: 0.5rem; -} - -.sectionTitle::before { - content: ''; - display: inline-block; - width: 4px; - height: 1em; - background: var(--accent-coral, #e07256); - border-radius: 2px; -} - -.chartContainer { - width: 100%; - height: 400px; - position: relative; -} - -.loadingMessage { - text-align: center; - padding: 3rem; - color: var(--text-secondary, #5c564d); - font-size: 1rem; -} - -/* Table Section */ -.tableSection { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; - padding: 2rem; - margin-bottom: 2rem; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); -} - -.tableWrapper { + gap: 0.75rem; overflow-x: auto; - max-width: 100%; - margin-top: 1rem; + padding: 0.75rem 0; + border-bottom: 1px solid var(--border-light, #e5dfd5); -webkit-overflow-scrolling: touch; } -/* Right-edge fade so phone users see the comparison table scrolls. - Otherwise the wider-than-viewport table silently clips. */ -@media (max-width: 640px) { - .tableWrapper { - -webkit-mask-image: linear-gradient(to right, #000 calc(100% - 28px), transparent); - mask-image: linear-gradient(to right, #000 calc(100% - 28px), transparent); - } +.schoolChip { + flex: 1 1 0; + min-width: 180px; + background: var(--bg-card, white); + border: 1px solid var(--border-light, #e5dfd5); + border-top: 3px solid var(--accent-coral, #e07256); + border-radius: 8px; + box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); + padding: 0.55rem 0.75rem; + display: flex; + gap: 0.55rem; + align-items: center; } -.comparisonTable { - width: 100%; - border-collapse: separate; - border-spacing: 0; - font-size: 0.9375rem; +.chipDot { + width: 11px; + height: 11px; + border-radius: 50%; + flex: none; } -.comparisonTable thead { - background: var(--bg-secondary, #f3ede4); +.chipText { + min-width: 0; } -.comparisonTable th { - padding: 1rem; - text-align: left; +.chipName { + display: block; font-weight: 600; + font-size: 0.92rem; + line-height: 1.25; color: var(--text-primary, #1a1612); - border-bottom: 2px solid var(--border-color, #e5dfd5); - background: var(--bg-secondary, #f3ede4); + text-decoration: none; +} + +.chipName:hover { + color: var(--accent-coral-dark, #b04a2e); +} + +.chipMeta { + display: block; + font-size: 0.78rem; + color: var(--text-muted, #6d685f); white-space: nowrap; - text-transform: uppercase; - font-size: 0.75rem; - letter-spacing: 0.05em; + overflow: hidden; + text-overflow: ellipsis; } -.comparisonTable td { - padding: 1rem; - border-bottom: 1px solid var(--border-color, #e5dfd5); - color: var(--text-secondary, #5c564d); - text-align: left; - background: var(--bg-card, white); -} - -/* Sticky first column (Year) so labels remain visible while scrolling */ -.comparisonTable th:first-child, -.comparisonTable td:first-child { - position: sticky; - left: 0; - z-index: 1; - box-shadow: 2px 0 4px -2px rgba(26, 22, 18, 0.08); -} - -.comparisonTable thead th:first-child { - z-index: 2; -} - -.comparisonTable tbody tr:hover td:first-child { +.chipRemove { + margin-left: auto; + border: none; background: var(--bg-secondary, #f3ede4); + color: var(--text-muted, #6d685f); + border-radius: 50%; + width: 22px; + height: 22px; + cursor: pointer; + flex: none; + font-size: 0.9rem; + line-height: 1; } -.comparisonTable tbody tr:last-child td { - border-bottom: none; -} - -.comparisonTable tbody tr:hover { - background: var(--bg-secondary, #f3ede4); -} - -.yearCell { - font-weight: 700; - color: var(--accent-gold, #c9a227); -} - -/* Empty State */ -.emptyState { - text-align: center; - padding: 4rem 2rem; - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; -} - -.emptyStateTitle { - font-size: 1.5rem; - font-weight: 600; - color: var(--text-primary, #1a1612); - margin-bottom: 0.5rem; - font-family: var(--font-playfair), 'Playfair Display', serif; -} - -.emptyStateDescription { - font-size: 1rem; - color: var(--text-secondary, #5c564d); - max-width: 400px; - margin: 0 auto 1.5rem; -} - -.metricDescription { - margin-top: 0.5rem; - font-size: 0.85rem; - color: var(--text-secondary); - max-width: 600px; - flex-basis: 100%; - margin-top: 0.25rem; -} - -.progressNote { - background: var(--bg-secondary); - border-left: 3px solid var(--accent-teal); - padding: 0.75rem 1rem; - margin: 0 0 1.5rem; - font-size: 0.875rem; - color: var(--text-secondary); - border-radius: 0 var(--radius-sm) var(--radius-sm) 0; -} - - -/* Responsive Design */ -@media (max-width: 768px) { - .headerContent { - flex-direction: column; - align-items: stretch; - } - - .header h1 { - font-size: 1.75rem; - } - - .metricSelector { - flex-direction: column; - align-items: stretch; - padding: 1rem; - border-radius: 8px; - } - - .metricSelect { - max-width: 100%; - } - - .schoolsGrid { - grid-template-columns: 1fr; - } - - .chartSection, - .tableSection { - padding: 1rem; - border-radius: 8px; - } - - .chartContainer { - /* Taller than desktop's proportion would suggest: the chip legend row - sits inside, and the in-chart title/legend/axis titles are gone, so - nearly all of this is plot area. */ - height: 340px; - } - - .comparisonTable { - font-size: 0.875rem; - } - - .comparisonTable th, - .comparisonTable td { - padding: 0.75rem 0.5rem; - } - - .latestValue { - margin-left: -1rem; - margin-right: -1rem; - margin-bottom: -1rem; - padding: 1rem; - border-radius: 0 0 8px 5px; - } +.footnote { + font-size: 0.78rem; + color: var(--text-muted, #6d685f); + margin-top: 2.5rem; + border-top: 1px solid var(--border-light, #e5dfd5); + padding-top: 1rem; + max-width: 75ch; } diff --git a/nextjs-app/components/ComparisonView.tsx b/nextjs-app/components/ComparisonView.tsx index e1f929d..4e26d89 100644 --- a/nextjs-app/components/ComparisonView.tsx +++ b/nextjs-app/components/ComparisonView.tsx @@ -1,47 +1,38 @@ /** - * ComparisonView Component - * Client-side comparison interface with phase tabs, charts, and tables + * ComparisonView — the parent-first compare screen: a sticky school bar and + * six sections (At a glance / Ofsted / Academics / Getting a place / Who + * goes there / Explore trends), every number anchored against the England + * average or the computed state-school benchmark with provenance-correct + * labels. Layout and copy follow the reviewed mockups + * (docs/superpowers/specs/mockups/). */ 'use client'; import { useEffect, useRef, useState } from 'react'; import { useRouter, usePathname, useSearchParams } from 'next/navigation'; -import dynamic from 'next/dynamic'; import { useComparison } from '@/hooks/useComparison'; -const ComparisonChart = dynamic( - () => import('./ComparisonChart').then((m) => m.ComparisonChart), - { ssr: false }, -); import { SchoolSearchModal } from './SchoolSearchModal'; import { EmptyState } from './EmptyState'; -import { LoadingSkeleton } from './LoadingSkeleton'; -import type { ComparisonData, MetricDefinition, School } from '@/lib/types'; -import { formatPercentage, formatProgress, formatAcademicYear, CHART_COLORS, CHART_TEXT_COLORS, schoolUrl } from '@/lib/utils'; +import { CompareAtAGlance } from './compare/CompareAtAGlance'; +import { CompareOfsted } from './compare/CompareOfsted'; +import { CompareAcademics } from './compare/CompareAcademics'; +import { CompareAdmissions } from './compare/CompareAdmissions'; +import { CompareCommunity } from './compare/CompareCommunity'; +import { TrendsExplorer, PRIMARY_CATEGORIES, SECONDARY_CATEGORIES } from './compare/TrendsExplorer'; +import type { + Benchmarks, + ComparisonData, + MetricDefinition, + NationalAverages, + School, +} from '@/lib/types'; +import { CHART_COLORS, schoolUrl } from '@/lib/utils'; import { fetchComparison } from '@/lib/api'; import { track } from '@/lib/analytics'; import styles from './ComparisonView.module.css'; -const PRIMARY_CATEGORIES = ['expected', 'higher', 'progress', 'average', 'gender', 'equity', 'context', 'absence', 'trends']; -const SECONDARY_CATEGORIES = ['gcse']; - -const PRIMARY_OPTGROUPS: { label: string; category: string }[] = [ - { label: 'Expected Standard', category: 'expected' }, - { label: 'Higher Standard', category: 'higher' }, - { label: 'Progress Scores', category: 'progress' }, - { label: 'Average Scores', category: 'average' }, - { label: 'Gender Performance', category: 'gender' }, - { label: 'Equity (Disadvantaged)', category: 'equity' }, - { label: 'School Context', category: 'context' }, - { label: 'Absence', category: 'absence' }, - { label: '3-Year Trends', category: 'trends' }, -]; - -const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [ - { label: 'GCSE Performance', category: 'gcse' }, -]; - interface ComparisonViewProps { initialData: Record | null; initialUrns: number[]; @@ -58,11 +49,13 @@ export function ComparisonView({ const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - const { selectedSchools, removeSchool, addSchool, replaceSchools, isInitialized } = useComparison(); + const { selectedSchools, removeSchool, replaceSchools, isInitialized } = useComparison(); const [selectedMetric, setSelectedMetric] = useState(initialMetric); const [isModalOpen, setIsModalOpen] = useState(false); const [comparisonData, setComparisonData] = useState(initialData); + const [nationalAverages, setNationalAverages] = useState(); + const [benchmarks, setBenchmarks] = useState(); const [shareConfirm, setShareConfirm] = useState(false); const [comparePhase, setComparePhase] = useState<'primary' | 'secondary'>('primary'); // Tracks whether the user has explicitly clicked a phase tab. @@ -77,18 +70,18 @@ export function ComparisonView({ if (!isInitialized) return; if (initialUrns.length > 0 && initialData) { const urlSchools = initialUrns - .map(urn => initialData[String(urn)]?.school_info) + .map((urn) => initialData[String(urn)]?.school_info) .filter((info): info is NonNullable => Boolean(info)); const sameSet = urlSchools.length === selectedSchools.length && - urlSchools.every(s => selectedSchools.some(sel => sel.urn === s.urn)); + urlSchools.every((s) => selectedSchools.some((sel) => sel.urn === s.urn)); if (urlSchools.length > 0 && !sameSet) { replaceSchools(urlSchools); } } }, [isInitialized]); // eslint-disable-line react-hooks/exhaustive-deps - // Sync URL with selected schools + // Sync URL with selected schools + metric, and (re)fetch the comparison. useEffect(() => { const urns = selectedSchools.map((s) => s.urn).join(','); const params = new URLSearchParams(searchParams); @@ -104,20 +97,23 @@ export function ComparisonView({ const newUrl = `${pathname}?${params.toString()}`; router.replace(newUrl, { scroll: false }); - // Fetch comparison data if (selectedSchools.length > 0) { fetchComparison(urns, { cache: 'no-store' }) .then((data) => { setComparisonData(data.comparison); + setNationalAverages(data.national_averages); + setBenchmarks(data.benchmarks); }) .catch((err) => { // Keep whatever we already have (SSR data or a previous fetch) rather - // than blanking the chart — a transient refetch failure shouldn't + // than blanking the page — a transient refetch failure shouldn't // destroy a working comparison the user is looking at. console.error('Failed to fetch comparison:', err); }); } else { setComparisonData(null); + setNationalAverages(undefined); + setBenchmarks(undefined); } }, [selectedSchools, selectedMetric, pathname, searchParams, router]); @@ -128,27 +124,22 @@ export function ComparisonView({ if (info?.rwm_expected_pct != null) return 'primary'; // Fallback: check yearly data const yearlyData = comparisonData?.[school.urn]?.yearly_data; - if (yearlyData?.some((d: any) => d.attainment_8_score != null)) return 'secondary'; + if (yearlyData?.some((d) => d.attainment_8_score != null)) return 'secondary'; return 'primary'; }; - const primarySchools = selectedSchools.filter(s => classifySchool(s) === 'primary'); - const secondarySchools = selectedSchools.filter(s => classifySchool(s) === 'secondary'); + const primarySchools = selectedSchools.filter((s) => classifySchool(s) === 'primary'); + const secondarySchools = selectedSchools.filter((s) => classifySchool(s) === 'secondary'); - // Auto-select tab with more schools and sync the metric to match the detected phase. - // This fixes the case where the URL carries a primary metric (e.g. rwm_expected_pct) - // but the shortlisted schools are secondary — the phase tab switches but the metric - // needs to follow, otherwise all secondary cards show "–" for a primary-only field. + // Auto-select tab with more schools and sync the metric to match the phase. useEffect(() => { if (!comparisonData || selectedSchools.length === 0) return; if (phaseLockedByUser.current) return; const newPhase = secondarySchools.length > primarySchools.length ? 'secondary' : 'primary'; setComparePhase(newPhase); - // Only reset the metric when it doesn't belong to the newly detected phase. - // This preserves a correct metric that came from the URL (e.g. metric=attainment_8_score). const phaseCategories = newPhase === 'secondary' ? SECONDARY_CATEGORIES : PRIMARY_CATEGORIES; const metricFitsPhase = metrics.some( - (m) => m.key === selectedMetric && phaseCategories.includes(m.category) + (m) => m.key === selectedMetric && phaseCategories.includes(m.category), ); if (!metricFitsPhase) { setSelectedMetric(newPhase === 'secondary' ? 'attainment_8_score' : 'rwm_expected_pct'); @@ -158,29 +149,24 @@ export function ComparisonView({ const handlePhaseChange = (phase: 'primary' | 'secondary') => { phaseLockedByUser.current = true; setComparePhase(phase); - const defaultMetric = phase === 'secondary' ? 'attainment_8_score' : 'rwm_expected_pct'; - setSelectedMetric(defaultMetric); + setSelectedMetric(phase === 'secondary' ? 'attainment_8_score' : 'rwm_expected_pct'); }; // compare_viewed: fire once after the page has its first selection. - // We watch `selectedSchools.length` going from 0 → ≥1 so the event is - // sent only when there's actual content to view, not for empty arrivals. const compareViewedRef = useRef(false); useEffect(() => { if (compareViewedRef.current) return; if (selectedSchools.length === 0) return; compareViewedRef.current = true; - const primaryCount = selectedSchools.filter(s => s.phase?.toLowerCase().includes('primary')).length; + const primaryCount = selectedSchools.filter((s) => + s.phase?.toLowerCase().includes('primary'), + ).length; const secondaryCount = selectedSchools.length - primaryCount; - const phaseMix = primaryCount === 0 ? 'all_secondary' : secondaryCount === 0 ? 'all_primary' : 'mixed'; + const phaseMix = + primaryCount === 0 ? 'all_secondary' : secondaryCount === 0 ? 'all_primary' : 'mixed'; track('compare_viewed', { school_count: selectedSchools.length, phase_mix: phaseMix }); }, [selectedSchools]); - const handleMetricChange = (metric: string) => { - track('compare_metric_changed', { metric, phase: comparePhase }); - setSelectedMetric(metric); - }; - const handleRemoveSchool = (urn: number) => { removeSchool(urn); track('compare_school_removed', { urn, from: 'compare' }); @@ -191,21 +177,22 @@ export function ComparisonView({ const count = selectedSchools.length; const shareData = { title: 'School comparison · SchoolCompare', - text: count > 0 - ? `Comparing ${count} school${count === 1 ? '' : 's'} on SchoolCompare` - : 'SchoolCompare', + text: + count > 0 + ? `Comparing ${count} school${count === 1 ? '' : 's'} on SchoolCompare` + : 'SchoolCompare', url, }; - // Prefer the native share sheet on platforms that support it (iOS / Android). - // canShare is feature-detected because Safari iOS exposes share() but - // some configurations refuse the payload. - if (typeof navigator !== 'undefined' && navigator.share && (!navigator.canShare || navigator.canShare(shareData))) { + if ( + typeof navigator !== 'undefined' && + navigator.share && + (!navigator.canShare || navigator.canShare(shareData)) + ) { try { await navigator.share(shareData); track('compare_shared', { method: 'native', school_count: count }); return; } catch (err) { - // User cancelled — bail silently. Any other error falls through to clipboard. if ((err as DOMException)?.name === 'AbortError') return; } } @@ -214,27 +201,22 @@ export function ComparisonView({ track('compare_shared', { method: 'clipboard', school_count: count }); setShareConfirm(true); setTimeout(() => setShareConfirm(false), 2000); - } catch { /* fallback: do nothing */ } + } catch { + /* fallback: do nothing */ + } }; const isPrimary = comparePhase === 'primary'; - const allowedCategories = isPrimary ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES; - const optgroups = isPrimary ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS; - const filteredMetrics = metrics.filter(m => allowedCategories.includes(m.category)); const activeSchools = isPrimary ? primarySchools : secondarySchools; - // Get metric definition - const currentMetricDef = metrics.find((m) => m.key === selectedMetric); - const metricLabel = currentMetricDef?.label || selectedMetric; - - // No schools selected if (selectedSchools.length === 0) { return (

Compare Schools

- Add schools to your comparison basket to see side-by-side performance data + Add schools to your comparison basket to see them side by side — inspection results, + academics, admissions and community.

@@ -252,39 +234,46 @@ export function ComparisonView({ ); } - // Build filtered comparison data for active phase + // Build filtered comparison data for the active phase const activeComparisonData: Record = {}; if (comparisonData) { - activeSchools.forEach(s => { + activeSchools.forEach((s) => { if (comparisonData[s.urn]) { activeComparisonData[s.urn] = comparisonData[s.urn]; } }); } - - // Get years for table - const years = - Object.keys(activeComparisonData).length > 0 - ? activeComparisonData[Object.keys(activeComparisonData)[0]].yearly_data.map((d) => d.year) - : []; + const hasData = Object.keys(activeComparisonData).length > 0; return (
- {/* Header */}

Compare Schools

- Comparing {selectedSchools.length} school{selectedSchools.length !== 1 ? 's' : ''} + {selectedSchools.length} school{selectedSchools.length !== 1 ? 's' : ''} side by side + — each number anchored against the England average so you can tell at a glance + what's typical and what stands out.

-
+
@@ -292,20 +281,22 @@ export function ComparisonView({
{/* Phase Tabs */} -
- - -
+ {secondarySchools.length > 0 && primarySchools.length > 0 && ( +
+ + +
+ )} {activeSchools.length === 0 ? ( ) : ( <> - {/* Metric Selector */} -
- - - {currentMetricDef?.description && ( -

{currentMetricDef.description}

- )} -
- - {/* Progress score explanation */} - {selectedMetric.includes('progress') && ( -

- Progress scores measure pupils' progress from KS1 to KS2. A score of 0 equals the national average; positive scores are above average. -

- )} - - {/* School Cards */} -
-
- {activeSchools.map((school, index) => ( -
- -

- {school.school_name} -

-
- {school.local_authority && ( - {school.local_authority} - )} - {school.school_type && ( - {school.school_type} - )} -
- - {/* Latest metric value */} - {activeComparisonData[school.urn] && ( -
-
{metricLabel}
- {/* Text uses the AA-dark variant; the swatch dot keeps the true series colour */} -
- - {(() => { - const yearlyData = activeComparisonData[school.urn].yearly_data; - if (yearlyData.length === 0) return '-'; - - const latestData = yearlyData[yearlyData.length - 1]; - const value = latestData[selectedMetric as keyof typeof latestData]; - - if (value === null || value === undefined) return '-'; - - if (selectedMetric.includes('progress')) { - return formatProgress(value as number); - } else if (selectedMetric.includes('pct') || selectedMetric.includes('rate')) { - return formatPercentage(value as number); - } else { - return typeof value === 'number' ? value.toFixed(1) : String(value); - } - })()} -
-
- )} -
- ))} -
-
- - {/* Comparison Chart */} - {Object.keys(activeComparisonData).length > 0 ? ( -
-

Performance Over Time

-
- + {activeSchools.map((school, index) => ( +
+
-
- ) : activeSchools.length > 0 ? ( -
- -
- ) : null} + ))} +
- {/* Comparison Table */} - {Object.keys(activeComparisonData).length > 0 && years.length > 0 && ( -
-

Detailed Comparison

-
- - - - - {activeSchools.map((school) => ( - - ))} - - - - {years.map((year) => ( - - - {activeSchools.map((school) => { - const schoolData = activeComparisonData[school.urn]; - if (!schoolData) return ; + {hasData && ( + <> + + + + + + - const yearData = schoolData.yearly_data.find((d) => d.year === year); - if (!yearData) return ; - - const value = yearData[selectedMetric as keyof typeof yearData]; - - if (value === null || value === undefined) { - return ; - } - - let displayValue: string; - if (selectedMetric.includes('progress')) { - displayValue = formatProgress(value as number); - } else if (selectedMetric.includes('pct') || selectedMetric.includes('rate')) { - displayValue = formatPercentage(value as number); - } else { - displayValue = typeof value === 'number' ? value.toFixed(1) : String(value); - } - - return ; - })} - - ))} - -
Year{school.school_name}
{formatAcademicYear(year)}---{displayValue}
-
-
+

+ Sources: DfE Compare School Performance (KS2/KS4 results), Ofsted inspection + outcomes, DfE school admissions data, school census. England averages for test + results are official DfE figures; other benchmarks are state-school averages + computed from our dataset. Following DfE practice, figures based on 5 or fewer + pupils are suppressed and shown as "no data". +

+ )} )} - {/* School Search Modal */} setIsModalOpen(false)} />
); diff --git a/nextjs-app/components/compare/TrendsExplorer.tsx b/nextjs-app/components/compare/TrendsExplorer.tsx index c4ffd64..6cd5a6e 100644 --- a/nextjs-app/components/compare/TrendsExplorer.tsx +++ b/nextjs-app/components/compare/TrendsExplorer.tsx @@ -7,7 +7,6 @@ 'use client'; -import { useState } from 'react'; import dynamic from 'next/dynamic'; import { progressBand } from '@/lib/compareLogic'; @@ -53,19 +52,20 @@ export function TrendsExplorer({ schools, data, metrics, - initialMetric, + metric, + onMetricChange, isPrimaryPhase, nationalAverages, }: { schools: School[]; data: Record; metrics: MetricDefinition[]; - initialMetric: string; + /** Controlled: the page owns the metric so the URL contract survives. */ + metric: string; + onMetricChange: (metric: string) => void; isPrimaryPhase: boolean; nationalAverages?: NationalAverages; }) { - const [metric, setMetric] = useState(initialMetric); - const allowedCategories = isPrimaryPhase ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES; const optgroups = isPrimaryPhase ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS; const filteredMetrics = metrics.filter((m) => allowedCategories.includes(m.category)); @@ -88,7 +88,7 @@ export function TrendsExplorer({ const handleMetricChange = (next: string) => { track('compare_metric_changed', { metric: next, phase: isPrimaryPhase ? 'primary' : 'secondary' }); - setMetric(next); + onMetricChange(next); }; const ciKeys = PROGRESS_CI[metric]; From 2f85b9c64760467315f20bd56dd28c2228ede36d Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:59:22 +0100 Subject: [PATCH 40/59] test(e2e): compare journeys for the parent-first redesign Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- e2e/tests/journeys.spec.ts | 47 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index cf68899..936dfe2 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -138,7 +138,7 @@ test('results map fullscreen falls back to an overlay on iOS', async ({ page }) await expect(openFs).toBeVisible(); }); -test('comparing two schools shows both side by side', async ({ page }) => { +test('comparing two schools shows the parent-first sections side by side', async ({ page }) => { // Collect two school URNs from search results, then load the share URL await searchByName(page, 'primary'); await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 }); @@ -152,6 +152,36 @@ test('comparing two schools shows both side by side', async ({ page }) => { // Both schools' detail links should render in the comparison view await expect(page.locator(`a[href*="${urns[0]}"]`).first()).toBeVisible({ timeout: 15_000 }); await expect(page.locator(`a[href*="${urns[1]}"]`).first()).toBeVisible(); + + // The parent-first sections render in order (data-invariant: headings only) + for (const heading of [ + 'At a glance', + 'Ofsted inspection', + /How (children|students) do academically/, + 'Who goes there', + 'Explore trends', + ]) { + await expect( + page.getByRole('heading', { name: heading }).first(), + ).toBeVisible({ timeout: 15_000 }); + } + + // Every number gets an anchor: at least one England-average tick or label + await expect(page.getByText(/England \d+/).first()).toBeVisible(); + + // Ofsted linkout goes to the school's provider page, never a report deep-link + const ofstedLink = page.getByRole('link', { name: /Ofsted page/i }).first(); + await expect(ofstedLink).toBeVisible(); + expect(await ofstedLink.getAttribute('href')).toMatch( + /reports\.ofsted\.gov\.uk\/provider\/21\/\d+/ + ); + + // A school never shows both an overall-grade badge AND report-card detail: + // "Report card" implies "no overall grade is given" copy is present too. + const reportCards = await page.getByText('Report card', { exact: true }).count(); + if (reportCards > 0) { + await expect(page.getByText(/no overall grade/i).first()).toBeVisible(); + } }); test('compare chart on mobile shows school chips with tap-to-focus', async ({ page }) => { @@ -170,9 +200,22 @@ test('compare chart on mobile shows school chips with tap-to-focus', async ({ pa expect(urns.length).toBeGreaterThanOrEqual(3); await page.goto(`/compare?urns=${urns[0]},${urns[1]},${urns[2]}`); + + // Mobile is measure-first: the At a glance section stacks all active-phase + // schools inside one flow — no horizontal swiping between school columns. + await expect( + page.getByRole('heading', { name: 'At a glance' }), + ).toBeVisible({ timeout: 15_000 }); + const body = page.locator('body'); + const bodyOverflowsX = await body.evaluate( + (el) => el.scrollWidth > el.clientWidth + 1, + ); + expect(bodyOverflowsX).toBe(false); + + // The trends chart still renders (inside the Explore trends section)… await expect(page.locator('canvas:visible').first()).toBeVisible({ timeout: 15_000 }); - // The mobile chart legend renders one chip per school in the active phase. + // …with the mobile chart legend chips and tap-to-focus behaviour intact. const chipGroup = page.getByRole('group', { name: /highlight a school/i }); const chips = chipGroup.getByRole('button'); await expect(chips.first()).toBeVisible({ timeout: 15_000 }); From d0e71e2cf006eb364053c1ba6ffe964d0aa50e73 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:46:12 +0100 Subject: [PATCH 41/59] feat(api): compare school_info carries GIAS facts for the community section Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- backend/app.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/app.py b/backend/app.py index df0df38..e01c4c5 100644 --- a/backend/app.py +++ b/backend/app.py @@ -706,6 +706,15 @@ async def compare_schools( "phase": latest.get("phase", ""), "attainment_8_score": float(latest["attainment_8_score"]) if pd.notna(latest.get("attainment_8_score")) else None, "rwm_expected_pct": float(latest["rwm_expected_pct"]) if pd.notna(latest.get("rwm_expected_pct")) else None, + # GIAS facts the compare "Who goes there" section needs + # (same fields the detail endpoint exposes) + "religious_denomination": convert_to_native(latest.get("religious_denomination")), + "age_range": convert_to_native(latest.get("age_range")), + "gender": convert_to_native(latest.get("gender")), + "has_sixth_form": convert_to_native(latest.get("has_sixth_form")), + "capacity": convert_to_native(latest.get("capacity")), + "gias_total_pupils": convert_to_native(latest.get("gias_total_pupils")), + "trust_name": convert_to_native(latest.get("trust_name")), }, "yearly_data": clean_for_json(school_data), **supplementary_by_urn.get(urn, dict(_EMPTY_SUPPLEMENTARY)), From 6dd9b04b50bee146682da87efad8fc8b526251c5 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 07:07:56 +0100 Subject: [PATCH 42/59] ci: re-run PR checks (AI review job errored without posting findings) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB From 52f8994401d3136d02ac7902e07e626b8b24876f Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 13:05:03 +0100 Subject: [PATCH 43/59] perf(api): persist KS4 national averages as a mart; stop per-request aggregation fact_ks4_national_averages is computed once at dbt build time (covered by the EES DAG's stg_ees_ks4+ selector). _national_averages_payload now reads both national-averages marts instead of scanning the performance dataframe per year on every /api/compare request (~250ms saved per call). Fallback for the deploy-before-DAG window computes the latest year only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- backend/app.py | 148 +++++++++--------- backend/models.py | 17 ++ backend/tests/test_national_averages_marts.py | 93 +++++++++++ .../transform/models/marts/_marts_schema.yml | 6 + .../marts/fact_ks4_national_averages.sql | 25 +++ 5 files changed, 219 insertions(+), 70 deletions(-) create mode 100644 backend/tests/test_national_averages_marts.py create mode 100644 pipeline/transform/models/marts/fact_ks4_national_averages.sql diff --git a/backend/app.py b/backend/app.py index e01c4c5..b935af8 100644 --- a/backend/app.py +++ b/backend/app.py @@ -772,93 +772,101 @@ async def get_la_averages(request: Request): return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}} +_KS2_NATIONAL_METRICS = [ + "rwm_expected_pct", "rwm_high_pct", + "reading_expected_pct", "writing_expected_pct", "maths_expected_pct", + "gps_expected_pct", "gps_high_pct", "science_expected_pct", + "reading_avg_score", "maths_avg_score", "gps_avg_score", + "reading_progress", "writing_progress", "maths_progress", + "overall_absence_pct", "persistent_absence_pct", + "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct", +] +_KS4_NATIONAL_METRICS = [ + "attainment_8_score", "progress_8_score", + "english_maths_standard_pass_pct", "english_maths_strong_pass_pct", + "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct", + "ebacc_avg_score", "gcse_grade_91_pct", +] + + def _national_averages_payload(df: pd.DataFrame) -> dict: """National-averages payload shared by /api/national-averages and - /api/compare. Official DfE KS2 figures come from the mart table; - KS4 figures are computed from our dataset (no DfE dataset yet).""" + /api/compare. + + Both series are persisted marts computed at import time: official DfE + KS2 figures (fact_ks2_national_averages) and dataset-computed KS4 + averages (fact_ks4_national_averages) — the API never aggregates the + performance dataframe per request. If the KS4 mart hasn't been built + yet (deploy lands before the next DAG run), fall back to computing the + latest year only — a single-year scan, never the historical loop. + """ if df.empty: return {"primary": {}, "secondary": {}} - ks2_metrics = [ - "rwm_expected_pct", "rwm_high_pct", - "reading_expected_pct", "writing_expected_pct", "maths_expected_pct", - "gps_expected_pct", "gps_high_pct", "science_expected_pct", - "reading_avg_score", "maths_avg_score", "gps_avg_score", - "reading_progress", "writing_progress", "maths_progress", - "overall_absence_pct", "persistent_absence_pct", - "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct", - ] - ks4_metrics = [ - "attainment_8_score", "progress_8_score", - "english_maths_standard_pass_pct", "english_maths_strong_pass_pct", - "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct", - "ebacc_avg_score", "gcse_grade_91_pct", - ] + latest_year = int(df["year"].max()) - def _means(sub_df, metric_list): + from . import database + from .models import Ks2NationalAverage, Ks4NationalAverage + + def _row_metrics(row, metric_list): out = {} for col in metric_list: - if col in sub_df.columns: - val = sub_df[col].dropna() - if len(val) > 0: - out[col] = round(float(val.mean()), 2) + val = getattr(row, col, None) + if val is not None: + out[col] = val return out - latest_year = int(df["year"].max()) - df_latest = df[df["year"] == latest_year] - - # Primary: schools where KS2 data is non-null - primary_df = df_latest[df_latest["rwm_expected_pct"].notna()] - # Secondary: schools where KS4 data is non-null - secondary_df = df_latest[df_latest["attainment_8_score"].notna()] - - latest_primary = _means(primary_df, ks2_metrics) - latest_secondary = _means(secondary_df, ks4_metrics) - - # Per-year KS2 primary averages: use official DfE figures from the mart table. - # Per-year KS4 secondary averages: computed from our dataset (no DfE dataset yet). - from . import database - from .models import Ks2NationalAverage - - by_year = [] + ks2_rows: list = [] + ks4_rows: list = [] db = None try: db = database.SessionLocal() - nat_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all() - # Build a lookup of computed secondary averages per year as fallback - secondary_by_year = {} - for yr in sorted(df["year"].dropna().unique()): - yr = int(yr) - df_yr = df[df["year"] == yr] - secondary_by_year[yr] = _means( - df_yr[df_yr["attainment_8_score"].notna()], ks4_metrics - ) - # Merge: official KS2 figures + computed KS4 figures per year - ks2_years = {r.year for r in nat_rows} - all_years = sorted(ks2_years | set(secondary_by_year.keys())) - nat_lookup = {r.year: r for r in nat_rows} - for yr in all_years: - primary_yr: dict = {} - if yr in nat_lookup: - r = nat_lookup[yr] - for col in ks2_metrics: - val = getattr(r, col, None) - if val is not None: - primary_yr[col] = val - by_year.append({ - "year": yr, - "primary": primary_yr, - "secondary": secondary_by_year.get(yr, {}), - }) + try: + ks2_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all() + except Exception: + db.rollback() + try: + ks4_rows = db.query(Ks4NationalAverage).order_by(Ks4NationalAverage.year).all() + except Exception: + db.rollback() + except Exception: + pass finally: if db is not None: db.close() - # Update latest_primary with official DfE figure for the latest year if available - if by_year: - latest_official = next((e["primary"] for e in reversed(by_year) if e["primary"]), None) - if latest_official: - latest_primary = latest_official + primary_by_year = {r.year: _row_metrics(r, _KS2_NATIONAL_METRICS) for r in ks2_rows} + secondary_by_year = {r.year: _row_metrics(r, _KS4_NATIONAL_METRICS) for r in ks4_rows} + + if not any(secondary_by_year.values()): + # KS4 mart missing/empty: compute the latest year only. + df_latest = df[df["year"] == latest_year] + sec = ( + df_latest[df_latest["attainment_8_score"].notna()] + if "attainment_8_score" in df_latest.columns + else df_latest.iloc[0:0] + ) + vals = {} + for col in _KS4_NATIONAL_METRICS: + if col in sec.columns: + v = sec[col].dropna() + if len(v) > 0: + vals[col] = round(float(v.mean()), 2) + if vals: + secondary_by_year[latest_year] = vals + + all_years = sorted(set(primary_by_year) | set(secondary_by_year)) + by_year = [ + { + "year": yr, + "primary": primary_by_year.get(yr, {}), + "secondary": secondary_by_year.get(yr, {}), + } + for yr in all_years + ] + + latest_primary = next((e["primary"] for e in reversed(by_year) if e["primary"]), {}) + latest_secondary = next((e["secondary"] for e in reversed(by_year) if e["secondary"]), {}) return { "year": latest_year, diff --git a/backend/models.py b/backend/models.py index 8f97a16..b2325d9 100644 --- a/backend/models.py +++ b/backend/models.py @@ -231,6 +231,23 @@ class FactFinance(Base): premises_cost_pct = Column(Float) +class Ks4NationalAverage(Base): + """Computed national KS4 averages (from our dataset) — one row per year.""" + __tablename__ = "fact_ks4_national_averages" + __table_args__ = MARTS + + year = Column(Integer, primary_key=True) + attainment_8_score = Column(Float) + progress_8_score = Column(Float) + english_maths_standard_pass_pct = Column(Float) + english_maths_strong_pass_pct = Column(Float) + ebacc_entry_pct = Column(Float) + ebacc_standard_pass_pct = Column(Float) + ebacc_strong_pass_pct = Column(Float) + ebacc_avg_score = Column(Float) + gcse_grade_91_pct = Column(Float) + + class Ks2NationalAverage(Base): """Official DfE KS2 national headline averages — one row per academic year.""" __tablename__ = "fact_ks2_national_averages" diff --git a/backend/tests/test_national_averages_marts.py b/backend/tests/test_national_averages_marts.py new file mode 100644 index 0000000..b08c6df --- /dev/null +++ b/backend/tests/test_national_averages_marts.py @@ -0,0 +1,93 @@ +"""_national_averages_payload reads persisted marts (computed at import +time) — it must never loop the dataframe per year. The only dataframe work +allowed is the single-latest-year KS4 fallback for the window between a +deploy and the next DAG run.""" + +import numpy as np +import pandas as pd +import pytest + +LATEST = 202425 + + +def _df(): + return pd.DataFrame( + [ + dict(year=202324, attainment_8_score=40.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=50.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=30.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=np.nan, rwm_expected_pct=80.0), + ] + ) + + +class _Ks2Row: + year = LATEST + rwm_expected_pct = 62.1 + gps_expected_pct = 72.0 + + +class _Ks4Row: + year = LATEST + attainment_8_score = 46.5 + progress_8_score = -0.02 + + +class _StubSession: + """Returns KS2 rows for the first query and KS4 rows for the second — + mirroring the payload's query order.""" + + def __init__(self): + self.calls = 0 + + def query(self, model): + self._model = model.__name__ + return self + + def order_by(self, *a): + return self + + def all(self): + return [_Ks2Row()] if self._model == "Ks2NationalAverage" else [_Ks4Row()] + + def close(self): + pass + + +class _Ks4MissingSession(_StubSession): + def all(self): + if self._model == "Ks4NationalAverage": + raise RuntimeError("relation does not exist") + return [_Ks2Row()] + + def rollback(self): + pass + + +@pytest.fixture() +def payload(monkeypatch): + from backend import app as app_module + from backend import database as database_module + + def _run(session_cls): + monkeypatch.setattr(database_module, "SessionLocal", session_cls) + return app_module._national_averages_payload(_df()) + + return _run + + +def test_ks4_averages_come_from_the_mart_not_the_dataframe(payload): + body = payload(_StubSession) + # Mart value (46.5), NOT the dataframe mean of (50+30)/2 = 40.0 + assert body["secondary"]["attainment_8_score"] == 46.5 + assert body["primary"]["rwm_expected_pct"] == 62.1 + assert body["by_year"][-1]["secondary"]["progress_8_score"] == -0.02 + + +def test_missing_ks4_mart_falls_back_to_latest_year_only(payload): + body = payload(_Ks4MissingSession) + # Fallback computes the latest year from the df: mean(50, 30) = 40.0 + assert body["secondary"]["attainment_8_score"] == 40.0 + # ...and only the latest year — no historical KS4 loop + ks4_years = [e["year"] for e in body["by_year"] if e["secondary"]] + assert ks4_years == [LATEST] diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index bd4072e..9fc7860 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -160,6 +160,12 @@ models: - name: year tests: [not_null, unique] + - name: fact_ks4_national_averages + description: Computed national KS4 averages (means across state schools in our dataset — not official DfE figures) — one row per academic year + columns: + - name: year + tests: [not_null, unique] + - name: fact_deprivation description: IDACI deprivation index — one row per URN columns: diff --git a/pipeline/transform/models/marts/fact_ks4_national_averages.sql b/pipeline/transform/models/marts/fact_ks4_national_averages.sql new file mode 100644 index 0000000..5022332 --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks4_national_averages.sql @@ -0,0 +1,25 @@ +{{ config(materialized='table') }} + +-- Mart: Computed national KS4 averages — one row per academic year. +-- Unlike fact_ks2_national_averages (official DfE figures), DfE publishes no +-- KS4 national-headline dataset we ingest yet, so these are means computed +-- across the state schools in our dataset. Computed once at build time so the +-- API never has to aggregate the full performance table per request. +-- Semantics match the API's previous per-request computation: rows where +-- attainment_8_score is non-null; per-column means ignore NULLs. + +select + year, + round(avg(attainment_8_score)::numeric, 2) as attainment_8_score, + round(avg(progress_8_score)::numeric, 2) as progress_8_score, + round(avg(english_maths_standard_pass_pct)::numeric, 2) as english_maths_standard_pass_pct, + round(avg(english_maths_strong_pass_pct)::numeric, 2) as english_maths_strong_pass_pct, + round(avg(ebacc_entry_pct)::numeric, 2) as ebacc_entry_pct, + round(avg(ebacc_standard_pass_pct)::numeric, 2) as ebacc_standard_pass_pct, + round(avg(ebacc_strong_pass_pct)::numeric, 2) as ebacc_strong_pass_pct, + round(avg(ebacc_avg_score)::numeric, 2) as ebacc_avg_score, + round(avg(gcse_grade_91_pct)::numeric, 2) as gcse_grade_91_pct +from {{ ref('fact_ks4_performance') }} +where attainment_8_score is not null +group by year +order by year From 619e3a1189a2a9ece1d4e7e96729acdbb04d59b0 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 13:06:52 +0100 Subject: [PATCH 44/59] perf(compare): fetch only on school-set changes; use SSR payload; parallel page fetches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Metric changes no longer refire /api/compare (the data is already client-side; the picker is presentational) — the fetch effect depends only on the URN set, with URL sync split into its own effect. - The initial client fetch is skipped when the SSR payload already covers the selected schools; national averages + benchmarks now arrive via SSR props so nothing is lost by skipping. - page.tsx fetches comparison and metrics in parallel. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- nextjs-app/app/compare/page.tsx | 28 +++++----- nextjs-app/components/ComparisonView.tsx | 66 ++++++++++++++++-------- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/nextjs-app/app/compare/page.tsx b/nextjs-app/app/compare/page.tsx index 31689a5..0b2b4e4 100644 --- a/nextjs-app/app/compare/page.tsx +++ b/nextjs-app/app/compare/page.tsx @@ -32,26 +32,24 @@ export default async function ComparePage({ searchParams }: ComparePageProps) { const selectedMetric = metricParam || 'rwm_expected_pct'; try { - // Fetch comparison data if URNs provided - let comparisonData = null; - if (urns.length > 0) { - try { - const response = await fetchComparison(urnsParam!); - comparisonData = response.comparison; - } catch (error) { - console.error('Failed to fetch comparison:', error); - } - } + // Fetch comparison + metrics in parallel — they are independent. + const [comparisonResponse, metricsResponse] = await Promise.all([ + urns.length > 0 + ? fetchComparison(urnsParam!).catch((error) => { + console.error('Failed to fetch comparison:', error); + return null; + }) + : Promise.resolve(null), + fetchMetrics(), + ]); - // Fetch available metrics - const metricsResponse = await fetchMetrics(); - - // Metrics is already an array const metricsArray = metricsResponse?.metrics || []; return ( | null; + initialNationalAverages?: NationalAverages; + initialBenchmarks?: Benchmarks; initialUrns: number[]; metrics: MetricDefinition[]; selectedMetric: string; @@ -42,6 +44,8 @@ interface ComparisonViewProps { export function ComparisonView({ initialData, + initialNationalAverages, + initialBenchmarks, initialUrns, metrics, selectedMetric: initialMetric, @@ -54,8 +58,10 @@ export function ComparisonView({ const [selectedMetric, setSelectedMetric] = useState(initialMetric); const [isModalOpen, setIsModalOpen] = useState(false); const [comparisonData, setComparisonData] = useState(initialData); - const [nationalAverages, setNationalAverages] = useState(); - const [benchmarks, setBenchmarks] = useState(); + const [nationalAverages, setNationalAverages] = useState( + initialNationalAverages, + ); + const [benchmarks, setBenchmarks] = useState(initialBenchmarks); const [shareConfirm, setShareConfirm] = useState(false); const [comparePhase, setComparePhase] = useState<'primary' | 'secondary'>('primary'); // Tracks whether the user has explicitly clicked a phase tab. @@ -81,13 +87,16 @@ export function ComparisonView({ } }, [isInitialized]); // eslint-disable-line react-hooks/exhaustive-deps - // Sync URL with selected schools + metric, and (re)fetch the comparison. + const urnKey = selectedSchools.map((s) => s.urn).join(','); + + // Sync the URL with the selection + metric. Pure navigation state — no + // fetching here: metric changes are presentational (the data is already + // client-side) and must not refire the comparison request. useEffect(() => { - const urns = selectedSchools.map((s) => s.urn).join(','); const params = new URLSearchParams(searchParams); - if (urns) { - params.set('urns', urns); + if (urnKey) { + params.set('urns', urnKey); } else { params.delete('urns'); } @@ -96,26 +105,41 @@ export function ComparisonView({ const newUrl = `${pathname}?${params.toString()}`; router.replace(newUrl, { scroll: false }); + }, [urnKey, selectedMetric, pathname, searchParams, router]); - if (selectedSchools.length > 0) { - fetchComparison(urns, { cache: 'no-store' }) - .then((data) => { - setComparisonData(data.comparison); - setNationalAverages(data.national_averages); - setBenchmarks(data.benchmarks); - }) - .catch((err) => { - // Keep whatever we already have (SSR data or a previous fetch) rather - // than blanking the page — a transient refetch failure shouldn't - // destroy a working comparison the user is looking at. - console.error('Failed to fetch comparison:', err); - }); - } else { + // Fetch only when the school set changes. The very first run is skipped + // when the SSR payload already covers the current set — no double-fetch + // of data the server just rendered. + const firstFetchRef = useRef(true); + useEffect(() => { + if (!urnKey) { setComparisonData(null); setNationalAverages(undefined); setBenchmarks(undefined); + return; } - }, [selectedSchools, selectedMetric, pathname, searchParams, router]); + + if (firstFetchRef.current) { + firstFetchRef.current = false; + const ssrUrns = new Set(Object.keys(initialData ?? {})); + const covered = urnKey.split(',').every((urn) => ssrUrns.has(urn)); + if (covered && ssrUrns.size > 0) return; + } + + fetchComparison(urnKey, { cache: 'no-store' }) + .then((data) => { + setComparisonData(data.comparison); + setNationalAverages(data.national_averages); + setBenchmarks(data.benchmarks); + }) + .catch((err) => { + // Keep whatever we already have (SSR data or a previous fetch) rather + // than blanking the page — a transient refetch failure shouldn't + // destroy a working comparison the user is looking at. + console.error('Failed to fetch comparison:', err); + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [urnKey]); // Classify schools by phase using comparison data const classifySchool = (school: School): 'primary' | 'secondary' => { From d2dc78aeb599e16b7ef5019be2b360b08df463bc Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 13:25:24 +0100 Subject: [PATCH 45/59] ci: re-run PR checks (AI review job errored without posting findings) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB From 43a2c4a6bc539b621f31655aec05ef319a25f343 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 22:34:20 +0100 Subject: [PATCH 46/59] fix(compare): show SSR data on refresh; drop dead per-page comparison fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refresh bug: on mount the basket is empty for a beat before it hydrates from the URL. The fetch effect nulled comparisonData on that transient empty urnKey, then the one-shot 'SSR covers it' skip suppressed the refetch — leaving the page blank on reload. The effect is now gated on isInitialized, never blanks on empty (the render already shows the empty state when nothing is selected), and decides fetch-vs-skip by whether it already holds each requested school's data (SSR or a prior fetch). Perf: useComparison ran a useSWR('/api/compare') whose result nothing consumed — dead weight that fired on every page (Navigation + Toast are global) whenever the basket was non-empty, and duplicated ComparisonView's own fetch on the compare page. Removed; the hook now exposes basket state only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../ComparisonView.refresh.test.tsx | 82 +++++++++++++++++++ nextjs-app/components/ComparisonView.tsx | 39 ++++----- nextjs-app/hooks/useComparison.ts | 50 ++--------- 3 files changed, 111 insertions(+), 60 deletions(-) create mode 100644 nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx diff --git a/nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx b/nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx new file mode 100644 index 0000000..90bdae9 --- /dev/null +++ b/nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx @@ -0,0 +1,82 @@ +/** + * Regression: on refresh, the compare page must show the SSR-rendered data. + * + * The basket hydrates from the URL a beat after mount (selectedSchools is + * empty for the first render), so the fetch effect must not blank the + * SSR payload during that window — and must not refetch data the server + * already provided. + */ + +import { render, screen, waitFor } from '@testing-library/react'; + +import { ComparisonView } from '@/components/ComparisonView'; +import { ComparisonProvider } from '@/context/ComparisonProvider'; +import type { ComparisonData, School } from '@/lib/types'; + +const fetchComparison = jest.fn(); +jest.mock('@/lib/api', () => ({ + fetchComparison: (...args: unknown[]) => fetchComparison(...args), +})); +jest.mock('@/lib/analytics', () => ({ track: jest.fn() })); + +function school(urn: number, name: string): School { + return { + urn, + school_name: name, + local_authority: 'Testshire', + school_type: 'Community school', + rwm_expected_pct: 80, + phase: 'Primary', + } as School; +} + +function data(urn: number, name: string): ComparisonData { + return { + school_info: school(urn, name), + yearly_data: [{ year: 202425, rwm_expected_pct: 80 }] as ComparisonData['yearly_data'], + ofsted: null, + census: null, + admissions: null, + admissions_history: [], + deprivation: null, + }; +} + +const INITIAL_DATA = { + '100': data(100, 'Alpha Primary'), + '200': data(200, 'Beta Primary'), +}; + +beforeEach(() => { + fetchComparison.mockReset(); +}); + +test('renders SSR data on refresh without wiping it or refetching', async () => { + render( + + + , + ); + + // Both SSR-provided schools appear (data was not blanked during hydration) + await waitFor(() => { + expect(screen.getAllByText('Alpha Primary').length).toBeGreaterThan(0); + }); + expect(screen.getAllByText('Beta Primary').length).toBeGreaterThan(0); + expect(screen.getByRole('heading', { name: 'At a glance' })).toBeInTheDocument(); + + // …and the client never refetched data the server already rendered. + expect(fetchComparison).not.toHaveBeenCalled(); +}); diff --git a/nextjs-app/components/ComparisonView.tsx b/nextjs-app/components/ComparisonView.tsx index 418fb33..dc013fd 100644 --- a/nextjs-app/components/ComparisonView.tsx +++ b/nextjs-app/components/ComparisonView.tsx @@ -107,24 +107,26 @@ export function ComparisonView({ router.replace(newUrl, { scroll: false }); }, [urnKey, selectedMetric, pathname, searchParams, router]); - // Fetch only when the school set changes. The very first run is skipped - // when the SSR payload already covers the current set — no double-fetch - // of data the server just rendered. - const firstFetchRef = useRef(true); - useEffect(() => { - if (!urnKey) { - setComparisonData(null); - setNationalAverages(undefined); - setBenchmarks(undefined); - return; - } + // Fetch when the school set changes, but only for schools we don't already + // have data for. This skips the refetch of SSR-rendered data on load AND + // avoids a network call when a school is merely removed. A ref holds the + // latest data so the effect can read it without re-running on every fetch. + // + // Correctness note: we must NOT null the data on a transient empty urnKey. + // On mount the basket is empty for a beat before it hydrates from the URL, + // and blanking here (then skipping the refetch because SSR "covers" the set) + // was leaving the page empty on refresh. The render already shows the empty + // state whenever `selectedSchools` is empty, so stale data for deselected + // schools is harmless — it's simply unused. + const comparisonDataRef = useRef(comparisonData); + comparisonDataRef.current = comparisonData; - if (firstFetchRef.current) { - firstFetchRef.current = false; - const ssrUrns = new Set(Object.keys(initialData ?? {})); - const covered = urnKey.split(',').every((urn) => ssrUrns.has(urn)); - if (covered && ssrUrns.size > 0) return; - } + useEffect(() => { + if (!isInitialized || !urnKey) return; + + const have = comparisonDataRef.current ?? {}; + const covered = urnKey.split(',').every((urn) => have[urn] != null); + if (covered) return; fetchComparison(urnKey, { cache: 'no-store' }) .then((data) => { @@ -138,8 +140,7 @@ export function ComparisonView({ // destroy a working comparison the user is looking at. console.error('Failed to fetch comparison:', err); }); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [urnKey]); + }, [urnKey, isInitialized]); // Classify schools by phase using comparison data const classifySchool = (school: School): 'primary' | 'secondary' => { diff --git a/nextjs-app/hooks/useComparison.ts b/nextjs-app/hooks/useComparison.ts index f7ed256..68913e8 100644 --- a/nextjs-app/hooks/useComparison.ts +++ b/nextjs-app/hooks/useComparison.ts @@ -1,50 +1,18 @@ /** - * Custom hook for managing school comparison state - * Uses shared context for real-time updates across components + * Custom hook for managing school comparison state. + * + * This hook is mounted on every page via the global Navigation and + * ComparisonToast, so it must stay cheap — it exposes basket state only. + * The compare page fetches `/api/compare` itself (ComparisonView); nothing + * ever read the comparison payload from here, so the previous per-page SWR + * fetch (which fired on every page whenever the basket was non-empty) was + * dead weight and has been removed. */ 'use client'; -import useSWR from 'swr'; -import { fetcher } from '@/lib/api'; import { useComparisonContext } from '@/context/ComparisonContext'; -import type { ComparisonResponse } from '@/lib/types'; export function useComparison() { - const { - selectedSchools, - addSchool, - removeSchool, - replaceSchools, - clearAll, - isSelected, - canAddMore, - isInitialized, - } = useComparisonContext(); - - // Fetch comparison data for selected schools - const urns = selectedSchools.map((s) => s.urn).join(','); - const { data, error, isLoading, mutate } = useSWR( - selectedSchools.length > 0 ? `/compare?urns=${urns}` : null, - fetcher, - { - revalidateOnFocus: false, - dedupingInterval: 10000, - } - ); - - return { - selectedSchools, - comparisonData: data?.comparison, - isLoading, - error, - addSchool, - removeSchool, - replaceSchools, - clearAll, - isSelected, - canAddMore, - isInitialized, - mutate, - }; + return useComparisonContext(); } From 315f1feede70bdf3101d2fdd305b3d06e037fdac Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 22:42:10 +0100 Subject: [PATCH 47/59] =?UTF-8?q?perf(api):=20batch=20supplementary=20quer?= =?UTF-8?q?ies=20=E2=80=94=20one=20per=20table,=20not=20five=20per=20schoo?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_supplementary_data ran ~5 sequential DB round-trips per URN, so /api/compare scaled at ~37ms/school (measured on staging: 1 school 155ms, 3 schools 220ms, 6 schools 340ms). get_supplementary_data_batch fetches each table once with WHERE urn IN (...) and groups in Python, collapsing 5*N round-trips to a constant 5. get_supplementary_data is now a thin wrapper so the detail endpoint is unchanged; the compare endpoint makes one batched call. Each table degrades independently on failure. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- backend/app.py | 5 +- backend/data_loader.py | 208 ++++++++++++++-------- backend/tests/test_compare_enrichment.py | 8 +- backend/tests/test_supplementary_batch.py | 111 ++++++++++++ 4 files changed, 253 insertions(+), 79 deletions(-) create mode 100644 backend/tests/test_supplementary_batch.py diff --git a/backend/app.py b/backend/app.py index b935af8..eccd542 100644 --- a/backend/app.py +++ b/backend/app.py @@ -30,6 +30,7 @@ from .data_loader import ( load_latest_school_data, geocode_single_postcode, get_supplementary_data, + get_supplementary_data_batch, search_schools_typesense, ) from .data_loader import get_data_info as get_db_info @@ -679,8 +680,10 @@ async def compare_schools( db = None try: db = database.SessionLocal() + # One query per table for all schools, not ~5 queries per school. + batch = get_supplementary_data_batch(db, urn_list) for urn in urn_list: - supp = get_supplementary_data(db, urn) + supp = batch.get(urn, {}) supplementary_by_urn[urn] = { key: supp.get(key, default) for key, default in _EMPTY_SUPPLEMENTARY.items() diff --git a/backend/data_loader.py b/backend/data_loader.py index 7d73205..e7a8cd1 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -662,92 +662,150 @@ def _admissions_row_dict(a) -> dict: } -def get_supplementary_data(db: Session, urn: int) -> dict: - """Fetch all supplementary data for a single school URN.""" - result = {} +def _census_dict(pc) -> dict: + return { + "year": pc.year, + "total_pupils": pc.total_pupils, + "female_pupils": pc.female_pupils, + "male_pupils": pc.male_pupils, + "fsm_pct": pc.fsm_pct, + "eal_pct": pc.eal_pct, + } - def safe_query(model, pk_field, latest_field=None): + +def _deprivation_dict(d) -> dict: + return { + "lsoa_code": d.lsoa_code, + "idaci_score": d.idaci_score, + "idaci_decile": d.idaci_decile, + } + + +def _finance_dict(f) -> dict: + return { + "year": f.year, + "per_pupil_spend": f.per_pupil_spend, + "staff_cost_pct": f.staff_cost_pct, + "teacher_cost_pct": f.teacher_cost_pct, + "support_staff_cost_pct": f.support_staff_cost_pct, + "premises_cost_pct": f.premises_cost_pct, + } + + +def _empty_supplementary() -> dict: + return { + "ofsted": None, + "census": None, + "admissions": None, + "admissions_history": [], + "sen_detail": None, + "phonics": None, + "deprivation": None, + "finance": None, + } + + +def get_supplementary_data_batch(db: Session, urns: list[int]) -> dict: + """Fetch supplementary data for many URNs with one query per table + (WHERE urn IN (...)) instead of ~5 queries per school, collapsing the + per-request round-trips from 5*N to a constant 5. Returns {urn: block} + with the same shape get_supplementary_data produces per URN. + + Each table is queried independently and failures degrade that table to + empty for every URN — a missing mart never blanks the others. + """ + urns = [int(u) for u in urns] + result = {urn: _empty_supplementary() for urn in urns} + if not urns: + return result + + def _safe(fn): try: - q = db.query(model).filter(getattr(model, pk_field) == urn) - if latest_field: - q = q.order_by(getattr(model, latest_field).desc()) - return q.first() + fn() except Exception as e: import logging - logging.getLogger(__name__).error("safe_query failed for %s: %s", model.__name__, e) + logging.getLogger(__name__).error("batch supplementary query failed: %s", e) db.rollback() - return None - # Latest Ofsted inspection - o = safe_query(FactOfstedInspection, "urn", "inspection_date") - result["ofsted"] = _ofsted_block(o, urn) if o else None - - # Census (latest year of fact_pupil_characteristics) - pc = safe_query(FactPupilCharacteristics, "urn", "year") - result["census"] = ( - { - "year": pc.year, - "total_pupils": pc.total_pupils, - "female_pupils": pc.female_pupils, - "male_pupils": pc.male_pupils, - "fsm_pct": pc.fsm_pct, - "eal_pct": pc.eal_pct, - } - if pc - else None - ) - - # Admissions — all years, oldest first (for the multi-year trend view). - try: - admissions_rows = ( - db.query(FactAdmissions) - .filter(FactAdmissions.urn == urn) - .order_by(FactAdmissions.year.asc()) + # Ofsted — latest inspection per URN. Ordered so the first row seen per + # URN is the most recent. + def _ofsted(): + rows = ( + db.query(FactOfstedInspection) + .filter(FactOfstedInspection.urn.in_(urns)) + .order_by(FactOfstedInspection.urn, FactOfstedInspection.inspection_date.desc()) .all() ) - except Exception as e: - import logging - logging.getLogger(__name__).error("admissions history query failed: %s", e) - db.rollback() - admissions_rows = [] + seen = set() + for o in rows: + if o.urn in seen: + continue + seen.add(o.urn) + result[o.urn]["ofsted"] = _ofsted_block(o, o.urn) + _safe(_ofsted) - history = [_admissions_row_dict(a) for a in admissions_rows] - result["admissions_history"] = history - # Keep the single latest-year object for backwards-compatible consumers - # (hero chips, etc.). - result["admissions"] = history[-1] if history else None + # Census — latest year per URN. + def _census(): + rows = ( + db.query(FactPupilCharacteristics) + .filter(FactPupilCharacteristics.urn.in_(urns)) + .order_by(FactPupilCharacteristics.urn, FactPupilCharacteristics.year.desc()) + .all() + ) + seen = set() + for pc in rows: + if pc.urn in seen: + continue + seen.add(pc.urn) + result[pc.urn]["census"] = _census_dict(pc) + _safe(_census) - # SEN detail — not available in current marts - result["sen_detail"] = None + # Admissions — all years per URN, oldest first (multi-year trend view). + def _admissions(): + rows = ( + db.query(FactAdmissions) + .filter(FactAdmissions.urn.in_(urns)) + .order_by(FactAdmissions.urn, FactAdmissions.year.asc()) + .all() + ) + history: dict = {urn: [] for urn in urns} + for a in rows: + history[a.urn].append(_admissions_row_dict(a)) + for urn, rows_for_urn in history.items(): + result[urn]["admissions_history"] = rows_for_urn + result[urn]["admissions"] = rows_for_urn[-1] if rows_for_urn else None + _safe(_admissions) - # Phonics — no school-level data on EES - result["phonics"] = None + # Deprivation — one row per URN. + def _deprivation(): + rows = ( + db.query(FactDeprivation) + .filter(FactDeprivation.urn.in_(urns)) + .all() + ) + for d in rows: + result[d.urn]["deprivation"] = _deprivation_dict(d) + _safe(_deprivation) - # Deprivation - d = safe_query(FactDeprivation, "urn") - result["deprivation"] = ( - { - "lsoa_code": d.lsoa_code, - "idaci_score": d.idaci_score, - "idaci_decile": d.idaci_decile, - } - if d - else None - ) - - # Finance (latest year) - f = safe_query(FactFinance, "urn", "year") - result["finance"] = ( - { - "year": f.year, - "per_pupil_spend": f.per_pupil_spend, - "staff_cost_pct": f.staff_cost_pct, - "teacher_cost_pct": f.teacher_cost_pct, - "support_staff_cost_pct": f.support_staff_cost_pct, - "premises_cost_pct": f.premises_cost_pct, - } - if f - else None - ) + # Finance — latest year per URN. + def _finance(): + rows = ( + db.query(FactFinance) + .filter(FactFinance.urn.in_(urns)) + .order_by(FactFinance.urn, FactFinance.year.desc()) + .all() + ) + seen = set() + for f in rows: + if f.urn in seen: + continue + seen.add(f.urn) + result[f.urn]["finance"] = _finance_dict(f) + _safe(_finance) return result + + +def get_supplementary_data(db: Session, urn: int) -> dict: + """Supplementary data for a single URN (thin wrapper over the batch).""" + return get_supplementary_data_batch(db, [urn])[int(urn)] diff --git a/backend/tests/test_compare_enrichment.py b/backend/tests/test_compare_enrichment.py index 072ddca..0ff671f 100644 --- a/backend/tests/test_compare_enrichment.py +++ b/backend/tests/test_compare_enrichment.py @@ -67,7 +67,9 @@ def client(monkeypatch): monkeypatch.setattr(app_module, "load_school_data", _two_primary_schools_df) monkeypatch.setattr( - app_module, "get_supplementary_data", lambda db, urn: dict(CANNED_SUPPLEMENTARY) + app_module, + "get_supplementary_data_batch", + lambda db, urns: {int(u): dict(CANNED_SUPPLEMENTARY) for u in urns}, ) monkeypatch.setattr(database_module, "SessionLocal", _StubSession) return TestClient(app_module.app, raise_server_exceptions=False) @@ -102,10 +104,10 @@ def test_top_level_national_averages_and_benchmarks(client): def test_supplementary_failure_degrades_not_500(client, monkeypatch): from backend import app as app_module - def _boom(db, urn): + def _boom(db, urns): raise RuntimeError("marts unavailable") - monkeypatch.setattr(app_module, "get_supplementary_data", _boom) + monkeypatch.setattr(app_module, "get_supplementary_data_batch", _boom) resp = client.get("/api/compare?urns=100140") assert resp.status_code == 200 school = resp.json()["comparison"]["100140"] diff --git a/backend/tests/test_supplementary_batch.py b/backend/tests/test_supplementary_batch.py new file mode 100644 index 0000000..0593034 --- /dev/null +++ b/backend/tests/test_supplementary_batch.py @@ -0,0 +1,111 @@ +"""get_supplementary_data_batch fetches one query per table for all URNs +(not ~5 per school) and returns the same per-URN block shape as the +single-URN function, picking the latest row per URN where relevant.""" + +import types + +from backend import data_loader +from backend.data_loader import get_supplementary_data_batch + + +class _FakeQuery: + """Records that a query ran and serves canned rows filtered by an in-list.""" + + def __init__(self, recorder, model_name, rows): + self._rec = recorder + self._model = model_name + self._rows = rows + + def filter(self, *args, **kwargs): + return self + + def order_by(self, *args, **kwargs): + return self + + def all(self): + self._rec.append(self._model) + return self._rows + + def first(self): + self._rec.append(self._model) + return self._rows[0] if self._rows else None + + +class _FakeSession: + def __init__(self, rows_by_model): + self.rows_by_model = rows_by_model + self.queries: list[str] = [] + + def query(self, model): + name = model.__name__ + return _FakeQuery(self.queries, name, self.rows_by_model.get(name, [])) + + def rollback(self): + pass + + +def _ofsted_row(urn, date, oe): + base = {f: None for f in ( + "framework", "inspection_type", "quality_of_education", "behaviour_attitudes", + "personal_development", "leadership_management", "early_years_provision", + "sixth_form_provision", "ungraded_outcome", "ungraded_grade", + "rc_safeguarding_met", "rc_inclusion", "rc_curriculum_teaching", "rc_achievement", + "rc_attendance_behaviour", "rc_personal_development", "rc_leadership_governance", + "rc_early_years", "rc_sixth_form", "report_url", + )} + base.update(urn=urn, inspection_date=types.SimpleNamespace(isoformat=lambda: date), + overall_effectiveness=oe, grade_source=None) + return types.SimpleNamespace(**base) + + +def _adm_row(urn, year): + return types.SimpleNamespace( + urn=urn, year=year, school_phase="Primary", places_offered=100, + total_applications=200, first_preference_applications=150, + first_preference_offers=140, first_preference_offer_pct=93.3, + oversubscription_ratio=1.5, oversubscribed=True, + total_offers=100, second_preference_offers=5, third_preference_offers=2, + cross_la_applications=10, cross_la_offers=3, + ) + + +def test_one_query_per_table_and_latest_row_per_urn(): + rows = { + # URN 1 has two Ofsted rows; the batch must keep the most recent (2023). + "FactOfstedInspection": [ + _ofsted_row(1, "2023-01-01", 2), + _ofsted_row(1, "2019-01-01", 3), + _ofsted_row(2, "2021-06-01", 1), + ], + "FactAdmissions": [_adm_row(1, 202526), _adm_row(1, 202627), _adm_row(2, 202627)], + "FactPupilCharacteristics": [], + "FactDeprivation": [], + "FactFinance": [], + } + session = _FakeSession(rows) + out = get_supplementary_data_batch(session, [1, 2]) + + # Exactly one query per table — five total, regardless of two URNs. + assert sorted(session.queries) == [ + "FactAdmissions", "FactDeprivation", "FactFinance", + "FactOfstedInspection", "FactPupilCharacteristics", + ] + + # Latest Ofsted kept per URN + assert out[1]["ofsted"]["overall_effectiveness"] == 2 + assert out[2]["ofsted"]["overall_effectiveness"] == 1 + + # Admissions history grouped per URN, latest exposed as `admissions` + assert [r["year"] for r in out[1]["admissions_history"]] == [202526, 202627] + assert out[1]["admissions"]["year"] == 202627 + assert out[2]["admissions_history"] == [{**out[2]["admissions_history"][0]}] + + # Empty tables degrade to the null block, not a crash + assert out[1]["census"] is None and out[1]["deprivation"] is None + + +def test_single_wrapper_matches_batch(monkeypatch): + session = _FakeSession({"FactOfstedInspection": [_ofsted_row(5, "2022-01-01", 2)]}) + single = data_loader.get_supplementary_data(session, 5) + assert single["ofsted"]["overall_effectiveness"] == 2 + assert single["admissions_history"] == [] From 090d5f7bec824e083d3252e2c6e636686016304d Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 22:54:32 +0100 Subject: [PATCH 48/59] ci: speed up Frontend Typecheck + Tests; cancel superseded PR runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The job does ~3s of real work (typecheck 1.4s + jest 1.3s) but installs 452 MB / 460 packages every run. Two changes: - Cache nextjs-app/node_modules keyed on the lockfile hash (OS + node major pinned) and skip npm ci entirely on a hit — deps change rarely, so most PR pushes now do zero install. On miss, npm ci runs with --prefer-offline --no-audit --no-fund. - Workflow-level concurrency with cancel-in-progress: a new commit (or an empty re-trigger) aborts the previous run instead of stacking a second full matrix. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .gitea/workflows/pr-checks.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index 36a067b..b7d3a11 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -5,6 +5,13 @@ on: branches: - main +# Cancel superseded runs: pushing a new commit to a PR (or an empty +# re-trigger) aborts the previous still-running checks instead of running +# a second full matrix alongside them. +concurrency: + group: pr-checks-${{ gitea.event.pull_request.number }} + cancel-in-progress: true + env: REGISTRY: privaterepo.sitaru.org BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend @@ -23,12 +30,22 @@ jobs: uses: actions/setup-node@v4 with: node-version: 22 - cache: npm - cache-dependency-path: nextjs-app/package-lock.json + + # Cache the resolved node_modules (452 MB / 460 packages) keyed on the + # lockfile. On a hit — the common case, since deps change rarely — the + # whole `npm ci` step is skipped, not just its download phase. The key + # pins OS + node major so we never restore incompatible native binaries. + - name: Cache node_modules + id: node-modules-cache + uses: actions/cache@v4 + with: + path: nextjs-app/node_modules + key: nextjs-node-modules-${{ runner.os }}-node22-${{ hashFiles('nextjs-app/package-lock.json') }} - name: Install dependencies + if: steps.node-modules-cache.outputs.cache-hit != 'true' working-directory: nextjs-app - run: npm ci + run: npm ci --prefer-offline --no-audit --no-fund - name: Typecheck working-directory: nextjs-app From 06e4898c30feaedc471f97aba28ddb0d61379f4d Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 23:12:23 +0100 Subject: [PATCH 49/59] test(e2e): pick two same-phase schools for the compare journey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compare page's phase tabs put all-through schools (which carry KS4 data) on the secondary tab, so comparing an all-through school with a pure primary splits them across tabs and only the active tab renders its link. The test picked the first two 'primary' search hits without guaranteeing same phase, so it flaked whenever a search returned an all-through school first (e.g. URN 137306). Now selects two pure-Primary URNs via the API — deterministic and data-invariant. Verified against staging: was a 15.6s timeout, now passes in ~1.8s. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- e2e/tests/journeys.spec.ts | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index 936dfe2..1fee453 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -19,6 +19,27 @@ function schoolLinks(page: Page) { return page.locator('a[href^="/school/"]'); } +/** + * Two URNs guaranteed to be pure-primary (same phase). The compare page's + * phase tabs split all-through schools (which carry KS4 data) onto the + * secondary tab, so picking two arbitrary "primary" search hits can land + * them on different tabs where only the active one renders. Selecting via + * the API by exact phase keeps both on the same tab. Data-invariant: uses + * whatever primaries the environment holds. + */ +async function twoPrimaryUrns(page: Page): Promise<[string, string]> { + const res = await page.request.get('/api/schools?search=primary&per_page=50'); + expect(res.ok()).toBeTruthy(); + const body = await res.json(); + const urns: string[] = (body.schools ?? []) + .filter((s: { phase?: string; rwm_expected_pct?: number | null }) => + s.phase === 'Primary' && s.rwm_expected_pct != null, + ) + .map((s: { urn: number }) => String(s.urn)); + expect(urns.length).toBeGreaterThanOrEqual(2); + return [urns[0], urns[1]]; +} + test('home page loads with hero search', async ({ page }) => { await page.goto('/'); await expect(page.locator('h1').first()).toBeVisible(); @@ -139,19 +160,13 @@ test('results map fullscreen falls back to an overlay on iOS', async ({ page }) }); test('comparing two schools shows the parent-first sections side by side', async ({ page }) => { - // Collect two school URNs from search results, then load the share URL - await searchByName(page, 'primary'); - await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 }); - const hrefs = await schoolLinks(page).evaluateAll((links) => - links.map((l) => (l as HTMLAnchorElement).getAttribute('href') || '') - ); - const urns = [...new Set(hrefs.map((h) => h.match(/\/school\/(\d+)/)?.[1]).filter(Boolean))]; - expect(urns.length).toBeGreaterThanOrEqual(2); + // Two same-phase (pure primary) schools so both stay on one tab. + const [urn0, urn1] = await twoPrimaryUrns(page); - await page.goto(`/compare?urns=${urns[0]},${urns[1]}`); + await page.goto(`/compare?urns=${urn0},${urn1}`); // Both schools' detail links should render in the comparison view - await expect(page.locator(`a[href*="${urns[0]}"]`).first()).toBeVisible({ timeout: 15_000 }); - await expect(page.locator(`a[href*="${urns[1]}"]`).first()).toBeVisible(); + await expect(page.locator(`a[href*="${urn0}"]`).first()).toBeVisible({ timeout: 15_000 }); + await expect(page.locator(`a[href*="${urn1}"]`).first()).toBeVisible(); // The parent-first sections render in order (data-invariant: headings only) for (const heading of [ From e4565e9f158721d4df6b918f2b065de82851f8d9 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 14 Jul 2026 23:24:39 +0100 Subject: [PATCH 50/59] fix(compare): give the trends chart a real height (was squashed to ~150px) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ComparisonChart runs Chart.js with maintainAspectRatio:false, so it sizes to its container's height — which must be definite. TrendsExplorer gave .chartBox a min-height, which doesn't resolve the chart wrapper's height:100%, so Chart.js fell back to its ~150px default: a squashed 8.6:1 sliver that didn't match the mockups. Set a definite height (420px desktop, 360px mobile where the chips row sits above the canvas). Verified on staging by patching the live height: canvas went from 1287x150 to 1287x392 (desktop) / 284 (mobile) — proper ~3:1 proportions matching the mockup, with the England dashed line, COVID/2021-22 gap and table all reading correctly. An e2e guard asserts the trends canvas is taller than 220px so the squash can't regress. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- e2e/tests/journeys.spec.ts | 7 ++++++- .../components/compare/TrendsExplorer.module.css | 14 +++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index 936dfe2..b0a9e96 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -213,7 +213,12 @@ test('compare chart on mobile shows school chips with tap-to-focus', async ({ pa expect(bodyOverflowsX).toBe(false); // The trends chart still renders (inside the Explore trends section)… - await expect(page.locator('canvas:visible').first()).toBeVisible({ timeout: 15_000 }); + const chartCanvas = page.locator('canvas:visible').first(); + await expect(chartCanvas).toBeVisible({ timeout: 15_000 }); + // …at a real height, not the squashed ~150px Chart.js fallback that + // appears when the container lacks a definite height. + const chartBox = await chartCanvas.boundingBox(); + expect(chartBox && chartBox.height).toBeGreaterThan(220); // …with the mobile chart legend chips and tap-to-focus behaviour intact. const chipGroup = page.getByRole('group', { name: /highlight a school/i }); diff --git a/nextjs-app/components/compare/TrendsExplorer.module.css b/nextjs-app/components/compare/TrendsExplorer.module.css index d6ddd4d..61f17fa 100644 --- a/nextjs-app/components/compare/TrendsExplorer.module.css +++ b/nextjs-app/components/compare/TrendsExplorer.module.css @@ -60,8 +60,20 @@ margin: 0 0 1rem; } +/* ComparisonChart runs Chart.js with maintainAspectRatio:false, so it fills + its container's height — which must be *definite*. A min-height alone does + not resolve the chart wrapper's height:100%, leaving Chart.js to fall back + to its ~150px default (a squashed sliver). Give it a real height. */ .chartBox { - min-height: 320px; + height: 420px; +} + +@media (max-width: 640px) { + /* Taller on mobile: the mobile-only school chips sit above the canvas and + wrap to two rows for 3+ schools, so the plot keeps a usable height. */ + .chartBox { + height: 360px; + } } .tableWrapper { From 3cb72d0a0f4057fbe3fdbd70bc8183ab51a0f6d2 Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 07:29:24 +0100 Subject: [PATCH 51/59] fix(compare): remove the year-by-year data table from Explore trends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mockup's Explore trends section is the measure picker + chart only — no data table. Removes the table (and the now-unused progressBand / band chip / formatMetricValue plumbing that only fed it). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../compare/TrendsExplorer.module.css | 31 -------- .../components/compare/TrendsExplorer.tsx | 77 ++----------------- 2 files changed, 5 insertions(+), 103 deletions(-) diff --git a/nextjs-app/components/compare/TrendsExplorer.module.css b/nextjs-app/components/compare/TrendsExplorer.module.css index 61f17fa..7c74367 100644 --- a/nextjs-app/components/compare/TrendsExplorer.module.css +++ b/nextjs-app/components/compare/TrendsExplorer.module.css @@ -75,34 +75,3 @@ height: 360px; } } - -.tableWrapper { - overflow-x: auto; - margin-top: 1.5rem; -} - -.table { - width: 100%; - border-collapse: collapse; - font-size: 0.9rem; -} - -.table th, -.table td { - text-align: left; - padding: 0.6rem 0.75rem; - border-bottom: 1px solid var(--border-light); -} - -.table th { - background: var(--bg-secondary); - font-size: 0.8rem; - text-transform: uppercase; - letter-spacing: 0.03em; - color: var(--text-secondary); -} - -.yearCell { - font-weight: 600; - white-space: nowrap; -} diff --git a/nextjs-app/components/compare/TrendsExplorer.tsx b/nextjs-app/components/compare/TrendsExplorer.tsx index 6cd5a6e..0a5f83e 100644 --- a/nextjs-app/components/compare/TrendsExplorer.tsx +++ b/nextjs-app/components/compare/TrendsExplorer.tsx @@ -1,19 +1,17 @@ /** * Explore trends — the full grouped metric catalogue (nothing from the old - * compare page is lost; spec §4's tier 3) driving the year-by-year chart - * with its England reference line, plus the year-by-year table. Progress - * metrics carry CI-based bands for the years DfE published them. + * compare page is lost; spec §4's tier 3) driving the year-by-year chart with + * its England reference line. Matches the mockup: a measure picker and the + * chart only (no data table). */ 'use client'; import dynamic from 'next/dynamic'; -import { progressBand } from '@/lib/compareLogic'; import type { ComparisonData, MetricDefinition, NationalAverages, School } from '@/lib/types'; -import { formatAcademicYear, formatMetricValue, metricKind } from '@/lib/utils'; import { track } from '@/lib/analytics'; -import { Chip, Section, sectionStyles as s } from './sectionShared'; +import { Section } from './sectionShared'; import styles from './TrendsExplorer.module.css'; const ComparisonChart = dynamic( @@ -40,14 +38,6 @@ const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [ export const PRIMARY_CATEGORIES = PRIMARY_OPTGROUPS.map((g) => g.category); export const SECONDARY_CATEGORIES = SECONDARY_OPTGROUPS.map((g) => g.category); -const PROGRESS_CI: Record = { - reading_progress: ['reading_progress_lower_ci', 'reading_progress_upper_ci'], - writing_progress: ['writing_progress_lower_ci', 'writing_progress_upper_ci'], - maths_progress: ['maths_progress_lower_ci', 'maths_progress_upper_ci'], -}; - -const BAND_LABEL = { above: 'Above average', average: 'Average', below: 'Below average' } as const; - export function TrendsExplorer({ schools, data, @@ -78,21 +68,11 @@ export function TrendsExplorer({ nationalByYear[entry.year] = block?.[metric] ?? null; } - const years = [ - ...new Set( - schools.flatMap( - (school) => data[String(school.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [], - ), - ), - ].sort((a, b) => a - b); - const handleMetricChange = (next: string) => { track('compare_metric_changed', { metric: next, phase: isPrimaryPhase ? 'primary' : 'secondary' }); onMetricChange(next); }; - const ciKeys = PROGRESS_CI[metric]; - return (
Progress scores measure pupils' progress from KS1 to KS2. A score of 0 equals the - national average. DfE stopped publishing KS2 progress after 2022/23 (no KS1 baseline); - bands use DfE's confidence intervals, not the raw score alone. + national average. DfE stopped publishing KS2 progress after 2022/23 (no KS1 baseline).

)} @@ -142,52 +121,6 @@ export function TrendsExplorer({ nationalByYear={nationalByYear} /> - - {years.length > 0 && ( -
- - - - - {schools.map((school) => ( - - ))} - - - - {years.map((year) => ( - - - {schools.map((school) => { - const row = data[String(school.urn)]?.yearly_data.find( - (d) => Math.trunc(d.year) === year, - ) as (Record & { year: number }) | undefined; - const value = row?.[metric]; - if (typeof value !== 'number') return ; - const band = ciKeys - ? progressBand( - value, - (row?.[ciKeys[0]] as number | null) ?? null, - (row?.[ciKeys[1]] as number | null) ?? null, - ) - : null; - return ( - - ); - })} - - ))} - -
Year{school.school_name}
{formatAcademicYear(year)}– - {formatMetricValue(value, metricKind(metric))}{' '} - {band && ( - - {BAND_LABEL[band]} - - )} -
-
- )}
From 66bc5523f6f1c5f6e5d5367529c0ff104b1518aa Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 07:50:01 +0100 Subject: [PATCH 52/59] fix(compare): mobile measure-first cards to match the mockup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grid sections (At a glance, Ofsted, Getting a place, Who goes there) collapsed generically on mobile — grey label pills, full names wrapping to 3 lines, no dots — making the page ~2x the mockup's height and 'significantly different' from the mobile design. Each measure is now wrapped in a that is display:contents on desktop (so the label + cells still flow into the shared aligned grid, unchanged) and a white card on mobile with compact [dot][short name] [value] rows — matching the mobile mockup. The sticky school bar becomes scrollable short-name pills on mobile too. Adds a shortName() util. Desktop layout is unchanged (display:contents dissolves the wrapper). Validated the card mechanism and real content shapes (report-card cell, badges, %+chip rows) via static previews at both widths. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../components/CompareOfsted.test.tsx | 8 ++ nextjs-app/__tests__/lib/utils.test.ts | 15 ++++ .../components/ComparisonView.module.css | 42 ++++++++++ nextjs-app/components/ComparisonView.tsx | 5 +- .../components/compare/CompareAdmissions.tsx | 18 +++-- .../components/compare/CompareAtAGlance.tsx | 17 ++-- .../components/compare/CompareCommunity.tsx | 41 ++++++---- .../components/compare/CompareOfsted.tsx | 26 +++--- .../compare/compareSections.module.css | 81 ++++++++++++++----- .../components/compare/sectionShared.tsx | 39 ++++++++- nextjs-app/lib/utils.ts | 18 +++++ 11 files changed, 250 insertions(+), 60 deletions(-) diff --git a/nextjs-app/__tests__/components/CompareOfsted.test.tsx b/nextjs-app/__tests__/components/CompareOfsted.test.tsx index 8df5602..849d255 100644 --- a/nextjs-app/__tests__/components/CompareOfsted.test.tsx +++ b/nextjs-app/__tests__/components/CompareOfsted.test.tsx @@ -95,4 +95,12 @@ describe('CompareOfsted', () => { expect(links).toHaveLength(3); expect(links[0]).toHaveAttribute('href', 'https://reports.ofsted.gov.uk/provider/21/1'); }); + + it('renders a per-measure mobile tag with the short school name', () => { + render(); + // Each measure repeats the schools, so the short name ("Graded" from + // "Graded School") appears once per measure (4) via the cell tag. + expect(screen.getAllByText('Graded').length).toBe(4); + expect(screen.getAllByText('Card').length).toBe(4); + }); }); diff --git a/nextjs-app/__tests__/lib/utils.test.ts b/nextjs-app/__tests__/lib/utils.test.ts index a1ebbbc..9fb9fba 100644 --- a/nextjs-app/__tests__/lib/utils.test.ts +++ b/nextjs-app/__tests__/lib/utils.test.ts @@ -10,6 +10,7 @@ import { debounce, buildOfstedListBadge, metricKind, + shortName, computeYBounds, } from '@/lib/utils'; @@ -223,3 +224,17 @@ describe('isProposedToClose', () => { expect(isProposedToClose({})).toBe(false); }); }); + +describe('shortName', () => { + it('drops the trailing establishment-type words', () => { + expect(shortName('Barclay Primary School')).toBe('Barclay'); + expect(shortName('Elmhurst Primary School')).toBe('Elmhurst'); + expect(shortName("St Mary's Catholic Primary School")).toBe("St Mary's"); + expect(shortName('Riverside Community Junior School')).toBe('Riverside'); + }); + + it('keeps a name that carries no type suffix, capping very long ones', () => { + expect(shortName('Beaver Road')).toBe('Beaver Road'); + expect(shortName('A'.repeat(30), 10)).toBe('AAAAAAAAA…'); + }); +}); diff --git a/nextjs-app/components/ComparisonView.module.css b/nextjs-app/components/ComparisonView.module.css index 633ba12..8ea0615 100644 --- a/nextjs-app/components/ComparisonView.module.css +++ b/nextjs-app/components/ComparisonView.module.css @@ -132,6 +132,11 @@ color: var(--accent-coral-dark, #b04a2e); } +/* Full name on desktop, short name on the compact mobile pills. */ +.chipNameShort { + display: none; +} + .chipMeta { display: block; font-size: 0.78rem; @@ -163,3 +168,40 @@ padding-top: 1rem; max-width: 75ch; } + +/* Mobile: the sticky school bar becomes compact, horizontally-scrollable + pills with short names (matching the mobile mockup) instead of full-width + cards whose names wrap to several lines. */ +@media (max-width: 640px) { + .schoolChip { + flex: 0 0 auto; + min-width: 0; + border-top-width: 2px; + border-radius: 999px; + padding: 0.35rem 0.7rem; + box-shadow: none; + } + + .chipName { + font-size: 0.85rem; + white-space: nowrap; + } + + .chipNameFull { + display: none; + } + + .chipNameShort { + display: inline; + } + + .chipMeta { + display: none; + } + + .chipRemove { + width: 18px; + height: 18px; + font-size: 0.75rem; + } +} diff --git a/nextjs-app/components/ComparisonView.tsx b/nextjs-app/components/ComparisonView.tsx index dc013fd..003658b 100644 --- a/nextjs-app/components/ComparisonView.tsx +++ b/nextjs-app/components/ComparisonView.tsx @@ -28,7 +28,7 @@ import type { NationalAverages, School, } from '@/lib/types'; -import { CHART_COLORS, schoolUrl } from '@/lib/utils'; +import { CHART_COLORS, schoolUrl, shortName } from '@/lib/utils'; import { fetchComparison } from '@/lib/api'; import { track } from '@/lib/analytics'; import styles from './ComparisonView.module.css'; @@ -349,7 +349,8 @@ export function ComparisonView({ /> - {school.school_name} + {school.school_name} + {shortName(school.school_name)} {[school.local_authority, school.school_type].filter(Boolean).join(' · ')} diff --git a/nextjs-app/components/compare/CompareAdmissions.tsx b/nextjs-app/components/compare/CompareAdmissions.tsx index aeaa3dc..9e0b4e4 100644 --- a/nextjs-app/components/compare/CompareAdmissions.tsx +++ b/nextjs-app/components/compare/CompareAdmissions.tsx @@ -10,7 +10,7 @@ import { summariseAdmissions } from '@/lib/compareLogic'; import type { ComparisonData, School } from '@/lib/types'; import { CHART_COLORS } from '@/lib/utils'; -import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; export function CompareAdmissions({ schools, @@ -49,9 +49,10 @@ export function CompareAdmissions({ } > - - Interest in the school - + {schools.map((school, i) => { const a = rows[i]; return ( @@ -68,7 +69,9 @@ export function CompareAdmissions({ ); })} - First-choice families offered a place + + + {schools.map((school, i) => { const summary = summariseAdmissions(rows[i]); return ( @@ -95,7 +98,9 @@ export function CompareAdmissions({ ); })} - What this means + + + {schools.map((school, i) => { const a = rows[i]; const summary = summariseAdmissions(a); @@ -118,6 +123,7 @@ export function CompareAdmissions({ ); })} + ); diff --git a/nextjs-app/components/compare/CompareAtAGlance.tsx b/nextjs-app/components/compare/CompareAtAGlance.tsx index a2f6605..64b61ea 100644 --- a/nextjs-app/components/compare/CompareAtAGlance.tsx +++ b/nextjs-app/components/compare/CompareAtAGlance.tsx @@ -15,7 +15,7 @@ import { type ReportCardSummary, } from '@/lib/compareLogic'; import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types'; -import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; function ReportCardChips({ summary }: { summary: ReportCardSummary }) { return ( @@ -69,7 +69,7 @@ export function CompareAtAGlance({ return (
- Latest Ofsted inspection + {schools.map((school, i) => { const display = ofstedDisplay(data[String(school.urn)]?.ofsted); return ( @@ -87,16 +87,16 @@ export function CompareAtAGlance({ ); })} + - - {isSecondary ? 'Attainment 8 score' : 'Children reaching the expected standard'} - {schools.map((school, i) => { const value = headlineValues[i]; return ( @@ -131,8 +131,9 @@ export function CompareAtAGlance({ ); })} + - Getting a place + {schools.map((school, i) => { const summary = summariseAdmissions(data[String(school.urn)]?.admissions); return ( @@ -148,8 +149,9 @@ export function CompareAtAGlance({ ); })} + - Size + {schools.map((school, i) => { const census = data[String(school.urn)]?.census; const pupils = census?.total_pupils ?? school.total_pupils ?? null; @@ -174,6 +176,7 @@ export function CompareAtAGlance({ ); })} +
); diff --git a/nextjs-app/components/compare/CompareCommunity.tsx b/nextjs-app/components/compare/CompareCommunity.tsx index 9ae9cc3..186c427 100644 --- a/nextjs-app/components/compare/CompareCommunity.tsx +++ b/nextjs-app/components/compare/CompareCommunity.tsx @@ -9,7 +9,7 @@ import { verdict } from '@/lib/compareLogic'; import type { Benchmarks, ComparisonData, School } from '@/lib/types'; -import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; function pctSplit(part: number | null | undefined, total: number | null | undefined): string | null { if (part == null || total == null || total === 0) return null; @@ -48,7 +48,7 @@ export function CompareCommunity({ how="The school's community, from the latest school census. State-school averages are computed from our dataset and shown for context — there's no “right” number here." > - Pupils on roll + {schools.map((school, i) => { const info = data[String(school.urn)]?.school_info as (School & { gias_total_pupils?: number | null; capacity?: number | null }) | undefined; const census = data[String(school.urn)]?.census; @@ -74,8 +74,9 @@ export function CompareCommunity({ ); })} + - Girls / boys + {schools.map((school, i) => { const census = data[String(school.urn)]?.census; const girls = pctSplit(census?.female_pupils, census?.total_pupils); @@ -86,10 +87,12 @@ export function CompareCommunity({ ); })} + - - Free school meals - + {schools.map((school, i) => { const fsm = data[String(school.urn)]?.census?.fsm_pct ?? null; return ( @@ -104,10 +107,12 @@ export function CompareCommunity({ ); })} + - - English as an additional language - + {schools.map((school, i) => { const eal = data[String(school.urn)]?.census?.eal_pct ?? null; return ( @@ -116,10 +121,12 @@ export function CompareCommunity({ ); })} + - - Extra learning support (SEN) - + {schools.map((school, i) => { const rows = data[String(school.urn)]?.yearly_data ?? []; let sen: number | null = null; @@ -143,8 +150,9 @@ export function CompareCommunity({ ); })} + - Faith character + {schools.map((school, i) => { const info = data[String(school.urn)]?.school_info; const faith = info?.religious_denomination; @@ -155,8 +163,9 @@ export function CompareCommunity({ ); })} + - Ages + {schools.map((school, i) => { const info = data[String(school.urn)]?.school_info; return ( @@ -165,8 +174,9 @@ export function CompareCommunity({ ); })} + - Run by + {schools.map((school, i) => { const info = data[String(school.urn)]?.school_info; const trust = info?.trust_name; @@ -177,6 +187,7 @@ export function CompareCommunity({ ); })} + ); diff --git a/nextjs-app/components/compare/CompareOfsted.tsx b/nextjs-app/components/compare/CompareOfsted.tsx index e148674..805b1af 100644 --- a/nextjs-app/components/compare/CompareOfsted.tsx +++ b/nextjs-app/components/compare/CompareOfsted.tsx @@ -13,7 +13,7 @@ import { type OfstedDisplay, } from '@/lib/compareLogic'; import type { ComparisonData, OfstedInspection, School } from '@/lib/types'; -import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; const GRADE_TONE: Record = { 1: 'good', @@ -157,14 +157,15 @@ export function CompareOfsted({ } > - Result + {schools.map((school, i) => ( ))} + - Inspected + {schools.map((school, i) => { const ofsted = data[String(school.urn)]?.ofsted; const age = yearsSince(ofsted?.inspection_date ?? null); @@ -176,9 +177,12 @@ export function CompareOfsted({ ); })} - - Judgement detail - + + + {schools.map((school, i) => { const ofsted = data[String(school.urn)]?.ofsted; return ( @@ -196,9 +200,12 @@ export function CompareOfsted({ ); })} - - Ofsted page - + + + {schools.map((school, i) => { const url = data[String(school.urn)]?.ofsted?.ofsted_page_url ?? @@ -211,6 +218,7 @@ export function CompareOfsted({ ); })} + ); diff --git a/nextjs-app/components/compare/compareSections.module.css b/nextjs-app/components/compare/compareSections.module.css index 895adae..5cd0b4b 100644 --- a/nextjs-app/components/compare/compareSections.module.css +++ b/nextjs-app/components/compare/compareSections.module.css @@ -31,43 +31,71 @@ margin-top: 1.25rem; } +/* Mobile base: each measure is a card; each cell is a school row led by a + colour dot + short name. `display: contents` at ≥761px dissolves the card + back into the shared grid. */ +.measure { + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 12px; + box-shadow: var(--shadow-soft); + padding: 0.75rem 0.85rem; + margin-bottom: 0.6rem; +} + .rowLabel { font-size: 0.85rem; font-weight: 600; - color: var(--text-secondary); + color: var(--text-primary); display: flex; align-items: center; gap: 0.35rem; - background: var(--bg-secondary); - border-radius: 6px; - padding: 0.4rem 0.6rem; - margin-top: 0.8rem; + padding: 0 0 0.1rem; } .cell { - padding: 0.4rem 0.6rem; + display: flex; + align-items: center; + gap: 0.5rem; + flex-wrap: wrap; + padding: 0.45rem 0; + border-top: 1px solid var(--border-light); + margin-top: 0.45rem; font-size: 0.95rem; } -.cell::before { - content: attr(data-school); - display: block; - font-size: 0.72rem; +.cellTag { + display: inline-flex; + align-items: center; + gap: 0.4rem; + width: 5rem; + flex: none; + font-size: 0.8rem; font-weight: 600; - color: var(--sc, var(--text-muted)); + color: var(--sc, var(--text-secondary)); +} + +.cellDot { + width: 9px; + height: 9px; + border-radius: 50%; + background: var(--dot, var(--text-muted)); + flex: none; } .big { - font-size: 1.35rem; + font-size: 1.05rem; font-weight: 700; font-variant-numeric: tabular-nums; } .small { display: block; + flex-basis: 100%; + padding-left: 5.5rem; font-size: 0.8rem; color: var(--text-muted); - margin-top: 0.1rem; + margin-top: -0.05rem; } .chip { @@ -197,20 +225,37 @@ gap: 0 0.75rem; } + /* Dissolve the per-measure card so its label + cells become grid items of + .grid, keeping columns aligned across every measure. */ + .measure { + display: contents; + } + + .cellTag { + display: none; + } + .rowLabel { - background: none; - border-radius: 0; - margin-top: 0; + color: var(--text-secondary); padding: 0.85rem 0.5rem 0.85rem 0; border-bottom: 1px solid var(--border-light); } .cell { + display: block; padding: 0.85rem 0.25rem; + border-top: none; border-bottom: 1px solid var(--border-light); + margin-top: 0; } - .cell::before { - content: none; + .big { + font-size: 1.35rem; + } + + .small { + flex-basis: auto; + padding-left: 0; + margin-top: 0.1rem; } } diff --git a/nextjs-app/components/compare/sectionShared.tsx b/nextjs-app/components/compare/sectionShared.tsx index 89b5d7a..0c27ea5 100644 --- a/nextjs-app/components/compare/sectionShared.tsx +++ b/nextjs-app/components/compare/sectionShared.tsx @@ -10,7 +10,7 @@ import type { CSSProperties, ReactNode } from 'react'; import type { School } from '@/lib/types'; -import { CHART_TEXT_COLORS } from '@/lib/utils'; +import { CHART_COLORS, CHART_TEXT_COLORS, shortName } from '@/lib/utils'; import styles from './compareSections.module.css'; export function Section({ @@ -61,6 +61,29 @@ export function RowLabel({ children, tip }: { children: ReactNode; tip?: string ); } +/** + * One measure = its row label plus a cell per school. `display: contents` on + * desktop (see CSS) makes these flow into the section grid as if this wrapper + * weren't here, keeping columns aligned across measures; on mobile the wrapper + * becomes a card so each measure reads as its own block. + */ +export function Measure({ + label, + tip, + children, +}: { + label: ReactNode; + tip?: string; + children: ReactNode; +}) { + return ( +
+ {label} + {children} +
+ ); +} + export function Cell({ school, index, @@ -73,9 +96,19 @@ export function Cell({ return (
+ {/* Mobile-only per-school tag (dot + short name); hidden on desktop, + where the column header identifies the school. */} + + {children}
); diff --git a/nextjs-app/lib/utils.ts b/nextjs-app/lib/utils.ts index fa1edf0..1e1305d 100644 --- a/nextjs-app/lib/utils.ts +++ b/nextjs-app/lib/utils.ts @@ -59,6 +59,24 @@ export function truncate(text: string, maxLength: number): string { return text.slice(0, maxLength).trim() + '...'; } +/** + * A compact school label for tight spaces (mobile compare rows, chip bars): + * drop the trailing establishment-type words so "Barclay Primary School" → + * "Barclay", "St Mary's Catholic Primary School" → "St Mary's". Falls back to + * a length-capped truncation for names that don't carry a type suffix. + */ +export function shortName(name: string, maxLength = 20): string { + let s = name + .replace( + /\s+(primary|junior|infant|nursery|community|foundation|catholic|academy|school|college)\b.*$/i, + '', + ) + .trim(); + if (!s) s = name; + if (s.length > maxLength) s = s.slice(0, maxLength - 1).trim() + '…'; + return s; +} + /** * Format a school's age range for display, e.g. "3-11" → "Ages 3–11". * Display-only — leaves the raw `age_range` field (used for sixth-form From f579630fab6c456e26a6984a3e8eebdbe3184202 Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 08:57:21 +0100 Subject: [PATCH 53/59] School name cutoff fix --- .../compare/compareSections.module.css | 19 +++++++++++-------- nextjs-app/lib/utils.ts | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/nextjs-app/components/compare/compareSections.module.css b/nextjs-app/components/compare/compareSections.module.css index 5cd0b4b..f152a52 100644 --- a/nextjs-app/components/compare/compareSections.module.css +++ b/nextjs-app/components/compare/compareSections.module.css @@ -55,24 +55,27 @@ .cell { display: flex; - align-items: center; - gap: 0.5rem; + align-items: baseline; + gap: 0.35rem 0.5rem; flex-wrap: wrap; - padding: 0.45rem 0; + padding: 0.5rem 0; border-top: 1px solid var(--border-light); - margin-top: 0.45rem; + margin-top: 0.5rem; font-size: 0.95rem; } +/* The school name gets its own full-width line above the value — real + school names are long and varied, so a fixed-width name column truncated + them ("Our Lady Queen of H…") or crowded the value. */ .cellTag { display: inline-flex; align-items: center; gap: 0.4rem; - width: 5rem; - flex: none; + flex-basis: 100%; font-size: 0.8rem; font-weight: 600; color: var(--sc, var(--text-secondary)); + margin-bottom: 0.15rem; } .cellDot { @@ -92,10 +95,9 @@ .small { display: block; flex-basis: 100%; - padding-left: 5.5rem; font-size: 0.8rem; color: var(--text-muted); - margin-top: -0.05rem; + margin-top: 0; } .chip { @@ -187,6 +189,7 @@ display: flex; gap: 0.3rem; flex-wrap: wrap; + flex-basis: 100%; margin-top: 0.3rem; } diff --git a/nextjs-app/lib/utils.ts b/nextjs-app/lib/utils.ts index 1e1305d..928ac74 100644 --- a/nextjs-app/lib/utils.ts +++ b/nextjs-app/lib/utils.ts @@ -65,7 +65,7 @@ export function truncate(text: string, maxLength: number): string { * "Barclay", "St Mary's Catholic Primary School" → "St Mary's". Falls back to * a length-capped truncation for names that don't carry a type suffix. */ -export function shortName(name: string, maxLength = 20): string { +export function shortName(name: string, maxLength = 32): string { let s = name .replace( /\s+(primary|junior|infant|nursery|community|foundation|catholic|academy|school|college)\b.*$/i, From fef83b3bf244a9bf3cb4afa75dbc433d8725014f Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 12:32:36 +0100 Subject: [PATCH 54/59] fix(compare): sticky school bar hidden behind the site header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Navigation header is position:sticky top:0 (z-index 1000). The school bar was also sticky top:0 (z-index 10), so when scrolled it pinned at the same top:0 *behind* the header — on mobile 57 of its 72px were covered, leaving only a sliver, so you couldn't see which schools were selected. Offset the bar's sticky top to the header height (65px desktop, 57px mobile — the Navigation breakpoint is also 640px) so it pins just below. e2e guard asserts the bar's sticky offset is at least the header height (verified it fails against the pre-fix build). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- e2e/tests/journeys.spec.ts | 13 +++++++++++++ nextjs-app/components/ComparisonView.module.css | 11 +++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index d4cc865..6ae1d16 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -227,6 +227,19 @@ test('compare chart on mobile shows school chips with tap-to-focus', async ({ pa ); expect(bodyOverflowsX).toBe(false); + // The sticky school bar must pin *below* the sticky site header, not at + // top:0 where the header covers it and the selected schools are hidden. + // Assert the sticky offset directly (robust — no scroll timing needed). + const barTop = await page + .locator('[class*="schoolBar"]') + .first() + .evaluate((el) => parseFloat(getComputedStyle(el).top)); + const headerHeight = await page + .locator('[class*="header"]') + .first() + .evaluate((el) => el.getBoundingClientRect().height); + expect(barTop).toBeGreaterThanOrEqual(headerHeight - 1); + // The trends chart still renders (inside the Explore trends section)… const chartCanvas = page.locator('canvas:visible').first(); await expect(chartCanvas).toBeVisible({ timeout: 15_000 }); diff --git a/nextjs-app/components/ComparisonView.module.css b/nextjs-app/components/ComparisonView.module.css index 8ea0615..4b301e4 100644 --- a/nextjs-app/components/ComparisonView.module.css +++ b/nextjs-app/components/ComparisonView.module.css @@ -80,10 +80,12 @@ } /* Sticky school bar — column identity while scrolling; horizontal scroll on - narrow screens */ + narrow screens. Offset by the sticky site header's height (Navigation is + position: sticky, top: 0) so this bar pins just below it instead of + sliding underneath and being hidden. Header ≈ 65px desktop / 57px mobile. */ .schoolBar { position: sticky; - top: 0; + top: 65px; z-index: 10; background: var(--bg-primary, #faf7f2); display: flex; @@ -173,6 +175,11 @@ pills with short names (matching the mobile mockup) instead of full-width cards whose names wrap to several lines. */ @media (max-width: 640px) { + /* The mobile Navigation header is shorter (≈57px). */ + .schoolBar { + top: 57px; + } + .schoolChip { flex: 0 0 auto; min-width: 0; From b4b0249a06da062a1a86bee9845c9e14fde41d5a Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 17:23:40 +0100 Subject: [PATCH 55/59] Fix Ofsted transitional inspections, phase tab exclusions, and FSM benchmark comparison --- backend/data_loader.py | 1 + backend/tests/test_benchmarks.py | 21 ++++++++------- nextjs-app/__tests__/lib/compareLogic.test.ts | 7 +++++ nextjs-app/components/ComparisonView.tsx | 26 +++++++++++-------- .../components/compare/CompareAtAGlance.tsx | 8 ++++++ .../components/compare/CompareCommunity.tsx | 11 ++++---- .../components/compare/CompareOfsted.tsx | 12 +++++++++ nextjs-app/lib/compareLogic.ts | 8 +++++- nextjs-app/lib/types.ts | 1 + 9 files changed, 69 insertions(+), 26 deletions(-) diff --git a/backend/data_loader.py b/backend/data_loader.py index e7a8cd1..332ce5e 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -577,6 +577,7 @@ def compute_benchmarks(df: pd.DataFrame) -> dict: "eal_pct": _median(sub, "eal_pct"), "sen_support_pct": _median(sub, "sen_support_pct"), "disadvantaged_pct": _median(sub, "disadvantaged_pct"), + "fsm_pct": _median(sub, "fsm_pct"), "median_pupils": median_pupils, } if with_disadvantaged: diff --git a/backend/tests/test_benchmarks.py b/backend/tests/test_benchmarks.py index 01d8893..50c8233 100644 --- a/backend/tests/test_benchmarks.py +++ b/backend/tests/test_benchmarks.py @@ -17,33 +17,33 @@ def _df(): # weighted = (40*100 + 60*300) / 400 = 55.0 ; unweighted mean = 50.0 dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=100, rwm_expected_disadvantaged_pct=40.0, eal_pct=10.0, - sen_support_pct=10.0, disadvantaged_pct=20.0, total_pupils=200), + sen_support_pct=10.0, disadvantaged_pct=20.0, fsm_pct=15.0, total_pupils=200), dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=300, rwm_expected_disadvantaged_pct=60.0, eal_pct=20.0, - sen_support_pct=14.0, disadvantaged_pct=24.0, total_pupils=280), + sen_support_pct=14.0, disadvantaged_pct=24.0, fsm_pct=17.0, total_pupils=280), dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=np.nan, rwm_expected_disadvantaged_pct=99.0, eal_pct=30.0, - sen_support_pct=18.0, disadvantaged_pct=30.0, total_pupils=300), + sen_support_pct=18.0, disadvantaged_pct=30.0, fsm_pct=19.0, total_pupils=300), dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=50, rwm_expected_disadvantaged_pct=np.nan, eal_pct=np.nan, - sen_support_pct=np.nan, disadvantaged_pct=np.nan, total_pupils=np.nan), + sen_support_pct=np.nan, disadvantaged_pct=np.nan, fsm_pct=np.nan, total_pupils=np.nan), dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=40, rwm_expected_disadvantaged_pct=np.nan, eal_pct=40.0, - sen_support_pct=20.0, disadvantaged_pct=40.0, total_pupils=350), + sen_support_pct=20.0, disadvantaged_pct=40.0, fsm_pct=21.0, total_pupils=350), dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=60, rwm_expected_disadvantaged_pct=np.nan, eal_pct=50.0, - sen_support_pct=22.0, disadvantaged_pct=44.0, total_pupils=400), + sen_support_pct=22.0, disadvantaged_pct=44.0, fsm_pct=23.0, total_pupils=400), # Two secondary schools (attainment_8 non-null) dict(year=LATEST, attainment_8_score=45.0, eligible_pupils=180, rwm_expected_disadvantaged_pct=np.nan, eal_pct=15.0, - sen_support_pct=12.0, disadvantaged_pct=22.0, total_pupils=1000), + sen_support_pct=12.0, disadvantaged_pct=22.0, fsm_pct=12.0, total_pupils=1000), dict(year=LATEST, attainment_8_score=50.0, eligible_pupils=200, rwm_expected_disadvantaged_pct=np.nan, eal_pct=25.0, - sen_support_pct=16.0, disadvantaged_pct=26.0, total_pupils=1200), + sen_support_pct=16.0, disadvantaged_pct=26.0, fsm_pct=14.0, total_pupils=1200), # An older-year primary row that must NOT influence anything dict(year=202324, attainment_8_score=np.nan, eligible_pupils=500, rwm_expected_disadvantaged_pct=1.0, eal_pct=99.0, - sen_support_pct=99.0, disadvantaged_pct=99.0, total_pupils=9999), + sen_support_pct=99.0, disadvantaged_pct=99.0, fsm_pct=99.0, total_pupils=9999), ] return pd.DataFrame(rows) @@ -59,6 +59,8 @@ def test_medians_ignore_nan_and_older_years(): assert b["year"] == LATEST # eal medians over [10,20,30,40,50] = 30 assert b["primary"]["eal_pct"] == 30.0 + # fsm medians over [15,17,19,21,23] = 19 + assert b["primary"]["fsm_pct"] == 19.0 # median pupils over [200,280,300,350,400] = 300 assert b["primary"]["median_pupils"] == 300 @@ -66,6 +68,7 @@ def test_medians_ignore_nan_and_older_years(): def test_secondary_block_has_no_disadvantaged_rwm(): b = compute_benchmarks(_df()) assert "disadvantaged_rwm_expected_pct" not in b["secondary"] + assert b["secondary"]["fsm_pct"] == 13.0 assert b["secondary"]["median_pupils"] == 1100 diff --git a/nextjs-app/__tests__/lib/compareLogic.test.ts b/nextjs-app/__tests__/lib/compareLogic.test.ts index 03c2432..e1308bf 100644 --- a/nextjs-app/__tests__/lib/compareLogic.test.ts +++ b/nextjs-app/__tests__/lib/compareLogic.test.ts @@ -126,6 +126,13 @@ describe('ofstedDisplay', () => { expect(ofstedDisplay(ofsted({})).kind).toBe('none'); }); + it('identifies transitional inspections without overall grades', () => { + const transitional = ofstedDisplay( + ofsted({ overall_effectiveness: null, inspection_date: '2024-11-05' }), + ); + expect(transitional.kind).toBe('transitional'); + }); + it('uses the four legacy grade words', () => { expect(OFSTED_LEGACY_GRADES).toEqual({ 1: 'Outstanding', diff --git a/nextjs-app/components/ComparisonView.tsx b/nextjs-app/components/ComparisonView.tsx index 003658b..65e6624 100644 --- a/nextjs-app/components/ComparisonView.tsx +++ b/nextjs-app/components/ComparisonView.tsx @@ -142,19 +142,23 @@ export function ComparisonView({ }); }, [urnKey, isInitialized]); - // Classify schools by phase using comparison data - const classifySchool = (school: School): 'primary' | 'secondary' => { + const primarySchools = selectedSchools.filter((school) => { const info = comparisonData?.[school.urn]?.school_info; - if (info?.attainment_8_score != null) return 'secondary'; - if (info?.rwm_expected_pct != null) return 'primary'; - // Fallback: check yearly data - const yearlyData = comparisonData?.[school.urn]?.yearly_data; - if (yearlyData?.some((d) => d.attainment_8_score != null)) return 'secondary'; - return 'primary'; - }; + const hasPrimaryData = + info?.rwm_expected_pct != null || + comparisonData?.[school.urn]?.yearly_data?.some((d) => d.rwm_expected_pct != null); + if (hasPrimaryData) return true; + return school.phase?.toLowerCase().includes('primary') || false; + }); - const primarySchools = selectedSchools.filter((s) => classifySchool(s) === 'primary'); - const secondarySchools = selectedSchools.filter((s) => classifySchool(s) === 'secondary'); + const secondarySchools = selectedSchools.filter((school) => { + const info = comparisonData?.[school.urn]?.school_info; + const hasSecondaryData = + info?.attainment_8_score != null || + comparisonData?.[school.urn]?.yearly_data?.some((d) => d.attainment_8_score != null); + if (hasSecondaryData) return true; + return school.phase?.toLowerCase().includes('secondary') || false; + }); // Auto-select tab with more schools and sync the metric to match the phase. useEffect(() => { diff --git a/nextjs-app/components/compare/CompareAtAGlance.tsx b/nextjs-app/components/compare/CompareAtAGlance.tsx index 64b61ea..59c9fe7 100644 --- a/nextjs-app/components/compare/CompareAtAGlance.tsx +++ b/nextjs-app/components/compare/CompareAtAGlance.tsx @@ -83,6 +83,14 @@ export function CompareAtAGlance({ {display.carriedForward && Grade carried forward} )} + {display.kind === 'transitional' && ( + <> + + No overall grade + + Sub-judgements only + + )} {display.kind === 'none' && No inspection in our dataset} ); diff --git a/nextjs-app/components/compare/CompareCommunity.tsx b/nextjs-app/components/compare/CompareCommunity.tsx index 186c427..6d39a3c 100644 --- a/nextjs-app/components/compare/CompareCommunity.tsx +++ b/nextjs-app/components/compare/CompareCommunity.tsx @@ -31,13 +31,14 @@ export function CompareCommunity({ const bench = isSecondary ? benchmarks?.secondary : benchmarks?.primary; const fsmChip = (value: number | null) => { - if (value == null || bench?.disadvantaged_pct == null) return null; - const v = verdict(value, bench.disadvantaged_pct, 3); + const anchor = bench?.fsm_pct ?? bench?.disadvantaged_pct ?? null; + if (value == null || anchor == null) return null; + const v = verdict(value, anchor, 3); return ( - {v === 'above' && 'Above the state-school average'} - {v === 'close' && 'About the state-school average'} - {v === 'below' && 'Below the state-school average'} + {v === 'above' && `Above the state-school average (${Math.round(anchor)}%)`} + {v === 'close' && `About the state-school average (${Math.round(anchor)}%)`} + {v === 'below' && `Below the state-school average (${Math.round(anchor)}%)`} ); }; diff --git a/nextjs-app/components/compare/CompareOfsted.tsx b/nextjs-app/components/compare/CompareOfsted.tsx index 805b1af..56ff359 100644 --- a/nextjs-app/components/compare/CompareOfsted.tsx +++ b/nextjs-app/components/compare/CompareOfsted.tsx @@ -51,6 +51,18 @@ function ResultCell({ display }: { display: OfstedDisplay }) { ); } + if (display.kind === 'transitional') { + return ( + <> + + No overall grade + + + Inspected under transitional framework (sub-judgements only) + + + ); + } return ( <> diff --git a/nextjs-app/lib/compareLogic.ts b/nextjs-app/lib/compareLogic.ts index 0916161..b4cb44f 100644 --- a/nextjs-app/lib/compareLogic.ts +++ b/nextjs-app/lib/compareLogic.ts @@ -102,6 +102,7 @@ export type OfstedDisplay = | { kind: 'none' } | { kind: 'graded'; grade: number; gradeLabel: string; carriedForward: false } | { kind: 'carried_forward'; grade: number; gradeLabel: string; carriedForward: true } + | { kind: 'transitional' } | { kind: 'report_card'; summary: ReportCardSummary }; export function ofstedDisplay( @@ -117,7 +118,12 @@ export function ofstedDisplay( const grade = ofsted.overall_effectiveness; const gradeLabel = grade != null ? OFSTED_LEGACY_GRADES[grade] : undefined; - if (grade == null || gradeLabel === undefined) return { kind: 'none' }; + if (grade == null || gradeLabel === undefined) { + if (ofsted.inspection_date) { + return { kind: 'transitional' }; + } + return { kind: 'none' }; + } if (ofsted.grade_source === 'ungraded_carried_forward') { return { kind: 'carried_forward', grade, gradeLabel, carriedForward: true }; diff --git a/nextjs-app/lib/types.ts b/nextjs-app/lib/types.ts index 3634f87..f82d907 100644 --- a/nextjs-app/lib/types.ts +++ b/nextjs-app/lib/types.ts @@ -357,6 +357,7 @@ export interface BenchmarkBlock { eal_pct: number | null; sen_support_pct: number | null; disadvantaged_pct: number | null; + fsm_pct?: number | null; median_pupils: number | null; /** Primary only — weighted by cohort size. */ disadvantaged_rwm_expected_pct?: number | null; From e74d3882ce78a141fa1a57daa3102d7a58852dc3 Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 17:38:02 +0100 Subject: [PATCH 56/59] Pass phase state to compare sub-components to prevent phase metrics override by multi-phase schools --- nextjs-app/components/ComparisonView.tsx | 3 +++ nextjs-app/components/compare/CompareAcademics.tsx | 4 +++- nextjs-app/components/compare/CompareAtAGlance.tsx | 4 +++- nextjs-app/components/compare/CompareCommunity.tsx | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/nextjs-app/components/ComparisonView.tsx b/nextjs-app/components/ComparisonView.tsx index 65e6624..600794e 100644 --- a/nextjs-app/components/ComparisonView.tsx +++ b/nextjs-app/components/ComparisonView.tsx @@ -379,6 +379,7 @@ export function ComparisonView({ data={activeComparisonData} nationalAverages={nationalAverages} benchmarks={benchmarks} + isSecondary={!isPrimary} /> ; nationalAverages?: NationalAverages; benchmarks?: Benchmarks; + isSecondary?: boolean; }) { const urns = schools.map((school) => school.urn); const schoolNames = schools.map((school) => school.school_name); - const isSecondary = schools.some( + const isSecondary = propIsSecondary !== undefined ? propIsSecondary : schools.some( (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, ); diff --git a/nextjs-app/components/compare/CompareAtAGlance.tsx b/nextjs-app/components/compare/CompareAtAGlance.tsx index 59c9fe7..f73f548 100644 --- a/nextjs-app/components/compare/CompareAtAGlance.tsx +++ b/nextjs-app/components/compare/CompareAtAGlance.tsx @@ -47,14 +47,16 @@ export function CompareAtAGlance({ data, nationalAverages, benchmarks, + isSecondary: propIsSecondary, }: { schools: School[]; data: Record; nationalAverages?: NationalAverages; benchmarks?: Benchmarks; + isSecondary?: boolean; }) { const urns = schools.map((school) => school.urn); - const isSecondary = schools.some( + const isSecondary = propIsSecondary !== undefined ? propIsSecondary : schools.some( (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, ); const headlineKey = isSecondary ? 'attainment_8_score' : 'rwm_expected_pct'; diff --git a/nextjs-app/components/compare/CompareCommunity.tsx b/nextjs-app/components/compare/CompareCommunity.tsx index 6d39a3c..9e14288 100644 --- a/nextjs-app/components/compare/CompareCommunity.tsx +++ b/nextjs-app/components/compare/CompareCommunity.tsx @@ -20,12 +20,14 @@ export function CompareCommunity({ schools, data, benchmarks, + isSecondary: propIsSecondary, }: { schools: School[]; data: Record; benchmarks?: Benchmarks; + isSecondary?: boolean; }) { - const isSecondary = schools.some( + const isSecondary = propIsSecondary !== undefined ? propIsSecondary : schools.some( (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, ); const bench = isSecondary ? benchmarks?.secondary : benchmarks?.primary; From 8abff7a0a1946d0f83498ad544444aa7c9522da5 Mon Sep 17 00:00:00 2001 From: Tudor Date: Wed, 15 Jul 2026 23:21:40 +0100 Subject: [PATCH 57/59] feat: ingest independent schools in Ofsted tap and dbt staging --- pipeline/meltano.yml | 3 + .../tap-uk-ofsted/tap_uk_ofsted/tap.py | 113 ++++++++++++++---- .../models/staging/stg_ofsted_inspections.sql | 8 +- 3 files changed, 99 insertions(+), 25 deletions(-) diff --git a/pipeline/meltano.yml b/pipeline/meltano.yml index cfcda23..856ac86 100644 --- a/pipeline/meltano.yml +++ b/pipeline/meltano.yml @@ -49,6 +49,9 @@ plugins: - name: mi_url kind: string description: Ofsted Management Information download URL + - name: independent_mi_url + kind: string + description: Ofsted Independent Schools Management Information download URL - name: tap-uk-fbit namespace: uk_fbit diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index eaacbd7..d58d7b5 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import datetime import io import re @@ -14,20 +15,28 @@ GOV_UK_PAGE = ( "monthly-management-information-ofsteds-school-inspections-outcomes" ) +INDEPENDENT_GOV_UK_PAGE = ( + "https://www.gov.uk/government/statistical-data-sets/" + "non-association-independent-schools-inspections-and-outcomes-management-information" +) + # Column name → internal field, in priority order (first match wins). # Handles both current and older file formats. COLUMN_PRIORITY = { "urn": ["URN", "Urn", "urn"], "inspection_date": [ "Inspection start date of latest OEIF graded inspection", + "Inspection start date of latest OEIF standard inspection", "Inspection start date", "Inspection date", ], "inspection_type": [ "Inspection type of latest OEIF graded inspection", + "Inspection type of latest OEIF standard inspection", "Inspection type", ], "event_type_grouping": [ + "Event type grouping of latest OEIF standard inspection", "Event type grouping", "Inspection type grouping", ], @@ -52,10 +61,12 @@ COLUMN_PRIORITY = { "Effectiveness of leadership and management", ], "early_years_provision": [ + "Latest OEIF early years provision (where applicable)", "Latest OEIF early years provision", "Early years provision (where applicable)", ], "sixth_form_provision": [ + "Latest OEIF sixth form provision (where applicable)", "Latest OEIF sixth form provision", "Sixth form provision (where applicable)", ], @@ -68,12 +79,7 @@ COLUMN_PRIORITY = { "ungraded_inspection_date": [ "Date of latest ungraded inspection", ], - # Report Card fields (post-Nov 2025 framework). Confirmed verbatim MI - # headers per diagnose_compare_gaps.py's Task 1(c) findings. No MI column - # currently exists for early-years or sixth-form report-card grades, so - # those two fields are deliberately omitted here (see schema below) -- - # they stay absent from every record, same as the existing `report_url` - # pattern for fields with no COLUMN_PRIORITY entry. + # Report Card fields (post-Nov 2025 framework). "rc_safeguarding_met": ["Safeguarding standards"], "rc_inclusion": ["Inclusion"], "rc_curriculum_teaching": ["Curriculum and teaching"], @@ -81,6 +87,13 @@ COLUMN_PRIORITY = { "rc_attendance_behaviour": ["Attendance and behaviour"], "rc_personal_development": ["Personal development and wellbeing"], "rc_leadership_governance": ["Leadership and governance"], + "rc_early_years": ["Early years (where applicable)"], + "rc_sixth_form": ["Post-16 provision (where applicable)"], + "report_url": [ + "Web Link (opens in new window)", + "Web link to Ofsted provider page", + "Web link", + ], } @@ -103,6 +116,51 @@ def discover_csv_url() -> str | None: return matches[0] if matches else None +def discover_independent_csv_url() -> str | None: + """Scrape GOV.UK page to find the latest independent schools MI CSV download link.""" + resp = requests.get(INDEPENDENT_GOV_UK_PAGE, timeout=30) + resp.raise_for_status() + # Look for CSV attachment links + csv_links = re.findall( + r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.csv)"', + resp.text, + ) + if not csv_links: + # Fall back to ODS + csv_links = re.findall( + r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.ods)"', + resp.text, + ) + + months = { + 'january': 1, 'february': 2, 'march': 3, 'april': 4, 'may': 5, 'june': 6, + 'july': 7, 'august': 8, 'september': 9, 'october': 10, 'november': 11, 'december': 12 + } + + parsed_links = [] + for link in csv_links: + normalized_link = link.lower().replace('-', '_') + if 'most_recent' not in normalized_link: + continue + + match = re.search(r'as_at_(\d{1,2})_([a-z]+)_(\d{4})', normalized_link) + if match: + day, month_str, year = match.groups() + month = months.get(month_str) + if month: + try: + dt = datetime(int(year), month, int(day)) + parsed_links.append((dt, link)) + except ValueError: + continue + + parsed_links.sort(reverse=True) + if parsed_links: + return parsed_links[0][1] + + return csv_links[0] if csv_links else None + + class OfstedInspectionsStream(Stream): """Stream: Ofsted inspection records.""" @@ -131,8 +189,6 @@ class OfstedInspectionsStream(Stream): th.Property("rc_attendance_behaviour", th.StringType), th.Property("rc_personal_development", th.StringType), th.Property("rc_leadership_governance", th.StringType), - # No MI column exists for these yet; declared for forward - # compatibility with the mart schema, always emitted as absent/NULL. th.Property("rc_early_years", th.StringType), th.Property("rc_sixth_form", th.StringType), th.Property("report_url", th.StringType), @@ -148,15 +204,8 @@ class OfstedInspectionsStream(Stream): break return mapping - def get_records(self, context): - import pandas as pd - - url = self.config.get("mi_url") or discover_csv_url() - if not url: - self.logger.error("Could not discover Ofsted MI download URL") - return - - self.logger.info("Downloading Ofsted MI: %s", url) + def _fetch_and_parse_url(self, url: str, pd) -> list[dict]: + """Download file and parse records.""" resp = requests.get(url, timeout=120) resp.raise_for_status() @@ -172,8 +221,6 @@ class OfstedInspectionsStream(Stream): lines = text.split("\n") header_idx = 0 for i, line in enumerate(lines[:20]): - # Match lines where URN appears as a CSV field (start or after comma), - # not as a substring of words like "turn" or "return". if re.search(r'(?:^|,)\s*URN\s*(?:,|$)', line): header_idx = i break @@ -190,8 +237,14 @@ class OfstedInspectionsStream(Stream): for _, row in df.iterrows(): record = {} + for key in self.schema["properties"].keys(): + record[key] = None + for field, col in col_map.items(): - record[field] = row.get(col, None) + val = row.get(col, None) + if val == 'NULL': + val = None + record[field] = val # Cast URN try: @@ -201,6 +254,25 @@ class OfstedInspectionsStream(Stream): yield record + def get_records(self, context): + import pandas as pd + + # 1. State-funded schools + state_url = self.config.get("mi_url") or discover_csv_url() + if state_url: + self.logger.info("Downloading Ofsted state-funded MI: %s", state_url) + yield from self._fetch_and_parse_url(state_url, pd) + else: + self.logger.error("Could not discover Ofsted state-funded MI download URL") + + # 2. Independent schools + ind_url = self.config.get("independent_mi_url") or discover_independent_csv_url() + if ind_url: + self.logger.info("Downloading Ofsted independent MI: %s", ind_url) + yield from self._fetch_and_parse_url(ind_url, pd) + else: + self.logger.error("Could not discover Ofsted independent MI download URL") + class TapUKOfsted(Tap): """Singer tap for UK Ofsted Management Information.""" @@ -209,6 +281,7 @@ class TapUKOfsted(Tap): config_jsonschema = th.PropertiesList( th.Property("mi_url", th.StringType, description="Direct URL to Ofsted MI file"), + th.Property("independent_mi_url", th.StringType, description="Direct URL to Ofsted Independent Schools MI file"), ).to_dict() def discover_streams(self): diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_inspections.sql index b4e06de..518b24a 100644 --- a/pipeline/transform/models/staging/stg_ofsted_inspections.sql +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.sql @@ -46,12 +46,10 @@ renamed as ( {{ parse_report_card_grade('rc_attendance_behaviour') }}::integer as rc_attendance_behaviour, {{ parse_report_card_grade('rc_personal_development') }}::integer as rc_personal_development, {{ parse_report_card_grade('rc_leadership_governance') }}::integer as rc_leadership_governance, - -- No MI column exists for these yet (see tap.py); the tap never - -- emits rc_early_years/rc_sixth_form, so these stay NULL. - null::integer as rc_early_years, - null::integer as rc_sixth_form, + {{ parse_report_card_grade('rc_early_years') }}::integer as rc_early_years, + {{ parse_report_card_grade('rc_sixth_form') }}::integer as rc_sixth_form, - report_url + nullif(trim(report_url), 'NULL') as report_url from source where urn is not null and ( From 95f10bf3522617551fd3b4507a6f7d3929028cf1 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 16 Jul 2026 08:29:59 +0100 Subject: [PATCH 58/59] fix: convert NaN/NULL to None and restore record properties structure in tap.py --- .../plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index d58d7b5..2c3471a 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -237,18 +237,15 @@ class OfstedInspectionsStream(Stream): for _, row in df.iterrows(): record = {} - for key in self.schema["properties"].keys(): - record[key] = None - for field, col in col_map.items(): val = row.get(col, None) - if val == 'NULL': + if pd.isna(val) or val == 'NULL': val = None record[field] = val # Cast URN try: - record["urn"] = int(record["urn"]) + record["urn"] = int(record.get("urn")) except (ValueError, KeyError, TypeError): continue From 609bb923d96aa5730131463ea35d5efdd404bf96 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 16 Jul 2026 08:54:14 +0100 Subject: [PATCH 59/59] fix: preserve literal 'NULL' strings for primary key columns --- pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index 2c3471a..289e356 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -239,7 +239,7 @@ class OfstedInspectionsStream(Stream): record = {} for field, col in col_map.items(): val = row.get(col, None) - if pd.isna(val) or val == 'NULL': + if pd.isna(val): val = None record[field] = val