diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 03f5a35..31bb6ea 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Deploy (staging -> E2E gate -> production) +name: Stage (build -> staging -> E2E gate) on: push: @@ -193,48 +193,5 @@ jobs: env: BASE_URL: ${{ secrets.STAGING_BASE_URL }} - promote-prod: - name: Promote to Production - runs-on: ubuntu-latest - needs: [e2e-staging] - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Gitea Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ gitea.actor }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Retag verified images as prod - run: | - SHORT_SHA="sha-$(echo "${{ gitea.sha }}" | cut -c1-7)" - for IMAGE in \ - "${REGISTRY}/${BACKEND_IMAGE_NAME}" \ - "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \ - "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do - # Keep a rollback pointer before moving :prod - docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true - docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}" - echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod" - done - - - name: Trigger production stack update - run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}" - - - name: Wait for production to become healthy - run: | - echo "Polling ${PROD_BASE_URL} for up to 5 minutes..." - for i in $(seq 1 60); do - if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then - echo "Production is up (attempt $i)" - exit 0 - fi - sleep 5 - done - echo "Production did not become healthy in time" >&2 - exit 1 - env: - PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }} +# Production deployment is a second, manual approval: see promote.yml +# ("Promote to Production (manual)") and docs/DEPLOY.md. diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index 36a067b..b7d3a11 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -5,6 +5,13 @@ on: branches: - main +# Cancel superseded runs: pushing a new commit to a PR (or an empty +# re-trigger) aborts the previous still-running checks instead of running +# a second full matrix alongside them. +concurrency: + group: pr-checks-${{ gitea.event.pull_request.number }} + cancel-in-progress: true + env: REGISTRY: privaterepo.sitaru.org BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend @@ -23,12 +30,22 @@ jobs: uses: actions/setup-node@v4 with: node-version: 22 - cache: npm - cache-dependency-path: nextjs-app/package-lock.json + + # Cache the resolved node_modules (452 MB / 460 packages) keyed on the + # lockfile. On a hit — the common case, since deps change rarely — the + # whole `npm ci` step is skipped, not just its download phase. The key + # pins OS + node major so we never restore incompatible native binaries. + - name: Cache node_modules + id: node-modules-cache + uses: actions/cache@v4 + with: + path: nextjs-app/node_modules + key: nextjs-node-modules-${{ runner.os }}-node22-${{ hashFiles('nextjs-app/package-lock.json') }} - name: Install dependencies + if: steps.node-modules-cache.outputs.cache-hit != 'true' working-directory: nextjs-app - run: npm ci + run: npm ci --prefer-offline --no-audit --no-fund - name: Typecheck working-directory: nextjs-app diff --git a/.gitea/workflows/promote.yml b/.gitea/workflows/promote.yml new file mode 100644 index 0000000..703c4ff --- /dev/null +++ b/.gitea/workflows/promote.yml @@ -0,0 +1,126 @@ +name: Promote to Production (manual) + +# Second approval gate of the deploy model: run this workflow from the +# Actions UI after testing the feature on staging. It refuses commits +# whose staging E2E gate is not green. See docs/DEPLOY.md. + +on: + workflow_dispatch: + inputs: + sha: + description: >- + Commit SHA on main to promote (full or >=7 chars). + Leave empty to promote the latest main commit. + required: false + default: "" + +# Only one promotion at a time; never cancel an in-flight promotion. +concurrency: + group: prod-promotion + cancel-in-progress: false + +env: + REGISTRY: privaterepo.sitaru.org + BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend + FRONTEND_IMAGE_NAME: ${{ gitea.repository }}-frontend + PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline + +jobs: + promote-prod: + name: Promote approved commit to Production + runs-on: ubuntu-latest + steps: + - name: Checkout repository (full history for ancestry check) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve and validate target SHA + id: resolve + # SECURITY: the dispatch input is untrusted — it reaches the shell + # only via env (never spliced into `run:` with ${{ }}) and is only + # used as a quoted argument. The resolved value is validated as a + # 40-hex sha and required to be an ancestor of main before any + # later step interpolates it. + env: + SHA_INPUT: ${{ gitea.event.inputs.sha }} + run: | + set -euo pipefail + case "$SHA_INPUT" in + -*) echo "REFUSED: SHA input may not start with '-'." >&2; exit 1 ;; + esac + if [ -z "$SHA_INPUT" ]; then + SHA_INPUT="$(git rev-parse origin/main)" + fi + FULL_SHA=$(git rev-parse --verify --quiet "${SHA_INPUT}^{commit}") || { + echo "REFUSED: not a commit in this repository." >&2 + exit 1 + } + echo "$FULL_SHA" | grep -Eq '^[0-9a-f]{40}$' + if ! git merge-base --is-ancestor "$FULL_SHA" origin/main; then + echo "REFUSED: $FULL_SHA is not on main — only main commits are promotable." >&2 + exit 1 + fi + SHORT_SHA="sha-$(echo "$FULL_SHA" | cut -c1-7)" + echo "full=$FULL_SHA" >> "$GITHUB_OUTPUT" + echo "short=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "Promoting $FULL_SHA (images tagged $SHORT_SHA)" + + - name: Verify the staging E2E gate passed for this commit + run: | + STATUS_JSON=$(curl -fsS \ + -H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \ + "https://${REGISTRY}/api/v1/repos/${{ gitea.repository }}/commits/${{ steps.resolve.outputs.full }}/status") + echo "$STATUS_JSON" | python3 -c " + import json, sys + d = json.load(sys.stdin) + ok = [s for s in d.get('statuses', []) + if 'E2E Journeys against Staging' in s.get('context', '') + and s.get('status') == 'success'] + if not ok: + print('REFUSED: no successful \"E2E Journeys against Staging\" status on this commit.') + print('Contexts found:', [s.get('context') for s in d.get('statuses', [])]) + sys.exit(1) + print('E2E gate verified green for this commit.') + " + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Gitea Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ gitea.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Retag approved images as prod (keeping rollback pointer) + run: | + SHORT_SHA="${{ steps.resolve.outputs.short }}" + for IMAGE in \ + "${REGISTRY}/${BACKEND_IMAGE_NAME}" \ + "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \ + "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do + # Keep a rollback pointer before moving :prod + docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true + docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}" + echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod" + done + + - name: Trigger production stack update + run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}" + + - name: Wait for production to become healthy + run: | + echo "Polling ${PROD_BASE_URL} for up to 5 minutes..." + for i in $(seq 1 60); do + if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then + echo "Production is up (attempt $i)" + exit 0 + fi + sleep 5 + done + echo "Production did not become healthy in time" >&2 + exit 1 + env: + PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }} diff --git a/backend/app.py b/backend/app.py index 1e7c758..eccd542 100644 --- a/backend/app.py +++ b/backend/app.py @@ -25,10 +25,12 @@ import asyncio from .config import settings from .data_loader import ( clear_cache, + compute_benchmarks, load_school_data, load_latest_school_data, geocode_single_postcode, get_supplementary_data, + get_supplementary_data_batch, search_schools_typesense, ) from .data_loader import get_data_info as get_db_info @@ -662,6 +664,36 @@ async def compare_schools( if comparison_data.empty: raise HTTPException(status_code=404, detail="No schools found") + # One session for all schools' supplementary blocks; failures degrade + # to empty blocks rather than failing a working comparison (mirrors + # the detail endpoint's defensive pattern). + from . import database + + _EMPTY_SUPPLEMENTARY = { + "ofsted": None, + "census": None, + "admissions": None, + "admissions_history": [], + "deprivation": None, + } + supplementary_by_urn: dict = {} + db = None + try: + db = database.SessionLocal() + # One query per table for all schools, not ~5 queries per school. + batch = get_supplementary_data_batch(db, urn_list) + for urn in urn_list: + supp = batch.get(urn, {}) + supplementary_by_urn[urn] = { + key: supp.get(key, default) + for key, default in _EMPTY_SUPPLEMENTARY.items() + } + except Exception: + supplementary_by_urn = {} + finally: + if db is not None: + db.close() + result = {} for urn in urn_list: school_data = comparison_data[comparison_data["urn"] == urn].sort_values("year") @@ -677,11 +709,27 @@ async def compare_schools( "phase": latest.get("phase", ""), "attainment_8_score": float(latest["attainment_8_score"]) if pd.notna(latest.get("attainment_8_score")) else None, "rwm_expected_pct": float(latest["rwm_expected_pct"]) if pd.notna(latest.get("rwm_expected_pct")) else None, + # GIAS facts the compare "Who goes there" section needs + # (same fields the detail endpoint exposes) + "religious_denomination": convert_to_native(latest.get("religious_denomination")), + "age_range": convert_to_native(latest.get("age_range")), + "gender": convert_to_native(latest.get("gender")), + "has_sixth_form": convert_to_native(latest.get("has_sixth_form")), + "capacity": convert_to_native(latest.get("capacity")), + "gias_total_pupils": convert_to_native(latest.get("gias_total_pupils")), + "trust_name": convert_to_native(latest.get("trust_name")), }, "yearly_data": clean_for_json(school_data), + **supplementary_by_urn.get(urn, dict(_EMPTY_SUPPLEMENTARY)), } - return {"comparison": result} + return { + "comparison": result, + # Official DfE anchors + computed state-school benchmarks so the + # compare UI can label provenance correctly (spec §8.6). + "national_averages": _national_averages_payload(df), + "benchmarks": compute_benchmarks(df), + } @app.get("/api/filters") @@ -727,96 +775,101 @@ async def get_la_averages(request: Request): return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}} -@app.get("/api/national-averages") -@limiter.limit(f"{settings.rate_limit_per_minute}/minute") -async def get_national_averages(request: Request): +_KS2_NATIONAL_METRICS = [ + "rwm_expected_pct", "rwm_high_pct", + "reading_expected_pct", "writing_expected_pct", "maths_expected_pct", + "gps_expected_pct", "gps_high_pct", "science_expected_pct", + "reading_avg_score", "maths_avg_score", "gps_avg_score", + "reading_progress", "writing_progress", "maths_progress", + "overall_absence_pct", "persistent_absence_pct", + "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct", +] +_KS4_NATIONAL_METRICS = [ + "attainment_8_score", "progress_8_score", + "english_maths_standard_pass_pct", "english_maths_strong_pass_pct", + "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct", + "ebacc_avg_score", "gcse_grade_91_pct", +] + + +def _national_averages_payload(df: pd.DataFrame) -> dict: + """National-averages payload shared by /api/national-averages and + /api/compare. + + Both series are persisted marts computed at import time: official DfE + KS2 figures (fact_ks2_national_averages) and dataset-computed KS4 + averages (fact_ks4_national_averages) — the API never aggregates the + performance dataframe per request. If the KS4 mart hasn't been built + yet (deploy lands before the next DAG run), fall back to computing the + latest year only — a single-year scan, never the historical loop. """ - Compute national average for each metric from the latest data year. - Returns separate averages for primary (KS2) and secondary (KS4) schools. - Values are derived from the loaded DataFrame so they automatically - stay current when new data is loaded. - """ - df = load_school_data() if df.empty: return {"primary": {}, "secondary": {}} - ks2_metrics = [ - "rwm_expected_pct", "rwm_high_pct", - "reading_expected_pct", "writing_expected_pct", "maths_expected_pct", - "reading_avg_score", "maths_avg_score", "gps_avg_score", - "reading_progress", "writing_progress", "maths_progress", - "overall_absence_pct", "persistent_absence_pct", - "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct", - ] - ks4_metrics = [ - "attainment_8_score", "progress_8_score", - "english_maths_standard_pass_pct", "english_maths_strong_pass_pct", - "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct", - "ebacc_avg_score", "gcse_grade_91_pct", - ] + latest_year = int(df["year"].max()) - def _means(sub_df, metric_list): + from . import database + from .models import Ks2NationalAverage, Ks4NationalAverage + + def _row_metrics(row, metric_list): out = {} for col in metric_list: - if col in sub_df.columns: - val = sub_df[col].dropna() - if len(val) > 0: - out[col] = round(float(val.mean()), 2) + val = getattr(row, col, None) + if val is not None: + out[col] = val return out - latest_year = int(df["year"].max()) - df_latest = df[df["year"] == latest_year] - - # Primary: schools where KS2 data is non-null - primary_df = df_latest[df_latest["rwm_expected_pct"].notna()] - # Secondary: schools where KS4 data is non-null - secondary_df = df_latest[df_latest["attainment_8_score"].notna()] - - latest_primary = _means(primary_df, ks2_metrics) - latest_secondary = _means(secondary_df, ks4_metrics) - - # Per-year KS2 primary averages: use official DfE figures from the mart table. - # Per-year KS4 secondary averages: computed from our dataset (no DfE dataset yet). - from .database import SessionLocal - from .models import Ks2NationalAverage - - by_year = [] + ks2_rows: list = [] + ks4_rows: list = [] + db = None try: - db = SessionLocal() - nat_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all() - # Build a lookup of computed secondary averages per year as fallback - secondary_by_year = {} - for yr in sorted(df["year"].dropna().unique()): - yr = int(yr) - df_yr = df[df["year"] == yr] - secondary_by_year[yr] = _means( - df_yr[df_yr["attainment_8_score"].notna()], ks4_metrics - ) - # Merge: official KS2 figures + computed KS4 figures per year - ks2_years = {r.year for r in nat_rows} - all_years = sorted(ks2_years | set(secondary_by_year.keys())) - nat_lookup = {r.year: r for r in nat_rows} - for yr in all_years: - primary_yr: dict = {} - if yr in nat_lookup: - r = nat_lookup[yr] - for col in ks2_metrics: - val = getattr(r, col, None) - if val is not None: - primary_yr[col] = val - by_year.append({ - "year": yr, - "primary": primary_yr, - "secondary": secondary_by_year.get(yr, {}), - }) + db = database.SessionLocal() + try: + ks2_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all() + except Exception: + db.rollback() + try: + ks4_rows = db.query(Ks4NationalAverage).order_by(Ks4NationalAverage.year).all() + except Exception: + db.rollback() + except Exception: + pass finally: - db.close() + if db is not None: + db.close() - # Update latest_primary with official DfE figure for the latest year if available - if by_year: - latest_official = next((e["primary"] for e in reversed(by_year) if e["primary"]), None) - if latest_official: - latest_primary = latest_official + primary_by_year = {r.year: _row_metrics(r, _KS2_NATIONAL_METRICS) for r in ks2_rows} + secondary_by_year = {r.year: _row_metrics(r, _KS4_NATIONAL_METRICS) for r in ks4_rows} + + if not any(secondary_by_year.values()): + # KS4 mart missing/empty: compute the latest year only. + df_latest = df[df["year"] == latest_year] + sec = ( + df_latest[df_latest["attainment_8_score"].notna()] + if "attainment_8_score" in df_latest.columns + else df_latest.iloc[0:0] + ) + vals = {} + for col in _KS4_NATIONAL_METRICS: + if col in sec.columns: + v = sec[col].dropna() + if len(v) > 0: + vals[col] = round(float(v.mean()), 2) + if vals: + secondary_by_year[latest_year] = vals + + all_years = sorted(set(primary_by_year) | set(secondary_by_year)) + by_year = [ + { + "year": yr, + "primary": primary_by_year.get(yr, {}), + "secondary": secondary_by_year.get(yr, {}), + } + for yr in all_years + ] + + latest_primary = next((e["primary"] for e in reversed(by_year) if e["primary"]), {}) + latest_secondary = next((e["secondary"] for e in reversed(by_year) if e["secondary"]), {}) return { "year": latest_year, @@ -826,6 +879,17 @@ async def get_national_averages(request: Request): } +@app.get("/api/national-averages") +@limiter.limit(f"{settings.rate_limit_per_minute}/minute") +async def get_national_averages(request: Request): + """ + National averages: official DfE KS2 figures per year plus computed + KS4 averages, derived from the loaded DataFrame and the + fact_ks2_national_averages mart. + """ + return _national_averages_payload(load_school_data()) + + @app.get("/api/metrics") @limiter.limit(f"{settings.rate_limit_per_minute}/minute") async def get_available_metrics(request: Request): diff --git a/backend/data_loader.py b/backend/data_loader.py index db72748..332ce5e 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -4,6 +4,7 @@ Provides efficient queries with caching. """ import logging +import re import pandas as pd import numpy as np @@ -20,6 +21,7 @@ from .models import ( FactOfstedInspection, FactAdmissions, FactDeprivation, FactFinance, FactPupilCharacteristics, ) +from .ofsted_codes import ofsted_page_url, report_card_labels from .schemas import SCHOOL_TYPE_MAP from .gias_codes import ( ADMISSIONS_POLICY, @@ -189,13 +191,20 @@ _MAIN_QUERY = text(""" p.reading_high_pct, p.reading_avg_score, p.reading_progress, + p.reading_progress_lower_ci, + p.reading_progress_upper_ci, p.writing_expected_pct, p.writing_high_pct, p.writing_progress, + p.writing_progress_lower_ci, + p.writing_progress_upper_ci, + p.writing_working_towards_pct, p.maths_expected_pct, p.maths_high_pct, p.maths_avg_score, p.maths_progress, + p.maths_progress_lower_ci, + p.maths_progress_upper_ci, p.gps_expected_pct, p.gps_high_pct, p.gps_avg_score, @@ -224,6 +233,9 @@ _MAIN_QUERY = text(""" p.progress_8_maths, p.progress_8_ebacc, p.progress_8_open, + p.progress_8_banding, + p.attainment_8_disadvantage_gap, + p.progress_8_disadvantage_gap, p.english_maths_strong_pass_pct, p.english_maths_standard_pass_pct, p.ebacc_entry_pct, @@ -262,25 +274,81 @@ assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), ( "expected replacement of 's.has_sixth_form,' to have taken effect" ) +# Fallback used when marts.dim_school predates the GIAS code-dictionary +# migration (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on +# this DB, so it still has the old name columns instead of *_code columns). +_MAIN_QUERY_LEGACY_NAMES = str(_MAIN_QUERY) +_LEGACY_NAME_REPLACEMENTS = [ + ("s.phase_code,", "s.phase,"), + ("s.school_type_code,", "s.school_type,"), + ( + "s.religious_character_code,", + "s.religious_character AS religious_denomination,", + ), + ("s.status_code,", "s.status,"), + ("s.admissions_policy_code,", "s.admissions_policy,"), +] +for _old, _new in _LEGACY_NAME_REPLACEMENTS: + assert _old in _MAIN_QUERY_LEGACY_NAMES, ( + f"expected {_old!r} to be present in _MAIN_QUERY before replacement" + ) + _MAIN_QUERY_LEGACY_NAMES = _MAIN_QUERY_LEGACY_NAMES.replace(_old, _new) +_MAIN_QUERY_LEGACY_NAMES = text(_MAIN_QUERY_LEGACY_NAMES) + +_GIAS_CODE_COLUMN_NAMES = ( + "phase_code", + "school_type_code", + "religious_character_code", + "status_code", + "admissions_policy_code", +) + +_MISSING_COLUMN_RE = re.compile(r'column "?(?:s\.)?(\w+)"? does not exist') + + +def _missing_column_name(exc: Exception) -> Optional[str]: + """Name of the missing column from a psycopg2 UndefinedColumn error. + + Inspects exc.orig (the DBAPI error), whose message names only the + offending column — str(exc) also embeds the full SQL statement, which + contains every column name and therefore must not be matched against. + """ + orig = getattr(exc, "orig", None) + match = _MISSING_COLUMN_RE.search(str(orig) if orig is not None else str(exc)) + return match.group(1) if match else None + def load_school_data_as_dataframe() -> pd.DataFrame: """Load all school + KS2 data as a pandas DataFrame.""" try: df = pd.read_sql(_MAIN_QUERY, engine) except sqlalchemy.exc.ProgrammingError as exc: - if "has_sixth_form" not in str(exc): + missing = _missing_column_name(exc) + if missing in _GIAS_CODE_COLUMN_NAMES: + logging.getLogger(__name__).warning( + "marts predate the GIAS code migration — falling back to " + "legacy name-column query: %s", + exc, + ) + try: + df = pd.read_sql(_MAIN_QUERY_LEGACY_NAMES, engine) + except Exception as exc2: + print(f"Warning: Could not load school data from marts: {exc2}") + return pd.DataFrame() + elif missing == "has_sixth_form": + logging.getLogger(__name__).warning( + "marts.dim_school is missing has_sixth_form (pipeline hasn't " + "rebuilt the mart yet on this DB) — retrying without it: %s", + exc, + ) + try: + df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine) + except Exception as exc2: + print(f"Warning: Could not load school data from marts: {exc2}") + return pd.DataFrame() + else: print(f"Warning: Could not load school data from marts: {exc}") return pd.DataFrame() - logging.getLogger(__name__).warning( - "marts.dim_school is missing has_sixth_form (pipeline hasn't " - "rebuilt the mart yet on this DB) — retrying without it: %s", - exc, - ) - try: - df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine) - except Exception as exc2: - print(f"Warning: Could not load school data from marts: {exc2}") - return pd.DataFrame() except Exception as exc: print(f"Warning: Could not load school data from marts: {exc}") return pd.DataFrame() @@ -457,138 +525,288 @@ def get_data_info(db: Session = None) -> dict: # SUPPLEMENTARY DATA — per-school detail page # ============================================================================= -def get_supplementary_data(db: Session, urn: int) -> dict: - """Fetch all supplementary data for a single school URN.""" - result = {} +def compute_benchmarks(df: pd.DataFrame) -> dict: + """State-school benchmarks computed from our dataset (spec §5/§8.6). - def safe_query(model, pk_field, latest_field=None): + NOT official DfE figures — consumers must label them + "state-school average (computed from our dataset)". The disadvantaged + attainment average is weighted by cohort size (eligible_pupils) so + small schools don't dominate; context measures are medians. + """ + if df.empty or "year" not in df.columns: + return {} + latest_year = df["year"].max() + if pd.isna(latest_year): + return {} + d = df[df["year"] == latest_year] + if d.empty: + return {} + is_secondary = ( + d["attainment_8_score"].notna() + if "attainment_8_score" in d.columns + else pd.Series(False, index=d.index) + ) + prim, sec = d[~is_secondary], d[is_secondary] + + def _median(sub, col): + if col not in sub.columns: + return None + v = sub[col].median() + return round(float(v), 1) if pd.notna(v) else None + + def _weighted_disadvantaged(sub): + needed = {"rwm_expected_disadvantaged_pct", "eligible_pupils"} + if not needed <= set(sub.columns): + return None + s = sub.dropna(subset=list(needed)) + if s.empty or s["eligible_pupils"].sum() == 0: + return None + w = ( + (s["rwm_expected_disadvantaged_pct"] * s["eligible_pupils"]).sum() + / s["eligible_pupils"].sum() + ) + return round(float(w), 1) + + def _block(sub, with_disadvantaged): + median_pupils = None + if "total_pupils" in sub.columns: + mp = sub["total_pupils"].median() + if pd.notna(mp): + median_pupils = int(mp) + block = { + "eal_pct": _median(sub, "eal_pct"), + "sen_support_pct": _median(sub, "sen_support_pct"), + "disadvantaged_pct": _median(sub, "disadvantaged_pct"), + "fsm_pct": _median(sub, "fsm_pct"), + "median_pupils": median_pupils, + } + if with_disadvantaged: + block["disadvantaged_rwm_expected_pct"] = _weighted_disadvantaged(sub) + return block + + return { + "source": "state-school average (computed from our dataset)", + "year": int(latest_year), + "primary": _block(prim, with_disadvantaged=True), + "secondary": _block(sec, with_disadvantaged=False), + } + + +def _ofsted_block(o, urn: int) -> dict: + """Serialize the latest Ofsted inspection row for API responses. + + `grade_source` records where the effective overall grade came from: + a graded (Section 5) inspection, or carried forward from an ungraded + (Section 8) outcome — materially different claims a UI must be able + to distinguish. `report_card` holds coded+labelled renewed-framework + (Nov 2025) area judgements; safeguarding is a separate boolean and + never appears among the graded areas. + """ + if o.overall_effectiveness is not None: + grade_source = "graded" + overall = o.overall_effectiveness + elif o.ungraded_grade is not None: + # Fall back to the grade parsed from an ungraded (Section 8) outcome + # (e.g. "School remains Good") so the detail page matches the list badge. + grade_source = "ungraded_carried_forward" + overall = o.ungraded_grade + else: + grade_source = None + overall = None + + block = { + "framework": o.framework, + "inspection_date": o.inspection_date.isoformat() if o.inspection_date else None, + "inspection_type": o.inspection_type, + "overall_effectiveness": overall, + "grade_source": grade_source, + "quality_of_education": o.quality_of_education, + "behaviour_attitudes": o.behaviour_attitudes, + "personal_development": o.personal_development, + "leadership_management": o.leadership_management, + "early_years_provision": o.early_years_provision, + "sixth_form_provision": o.sixth_form_provision, + "previous_overall": None, # Not available in new schema + "rc_safeguarding_met": o.rc_safeguarding_met, + "rc_inclusion": o.rc_inclusion, + "rc_curriculum_teaching": o.rc_curriculum_teaching, + "rc_achievement": o.rc_achievement, + "rc_attendance_behaviour": o.rc_attendance_behaviour, + "rc_personal_development": o.rc_personal_development, + "rc_leadership_governance": o.rc_leadership_governance, + "rc_early_years": o.rc_early_years, + "rc_sixth_form": o.rc_sixth_form, + "report_url": o.report_url, + "ofsted_page_url": ofsted_page_url(urn), + } + block["report_card"] = report_card_labels(block) + return block + + +def _admissions_row_dict(a) -> dict: + """Serialize one fact_admissions row for API responses.""" + return { + "year": a.year, + "school_phase": a.school_phase, + "places_offered": a.places_offered, + "total_applications": a.total_applications, + "first_preference_applications": a.first_preference_applications, + "first_preference_offers": a.first_preference_offers, + "first_preference_offer_pct": a.first_preference_offer_pct, + "oversubscription_ratio": a.oversubscription_ratio, + "oversubscribed": a.oversubscribed, + "total_offers": a.total_offers, + "second_preference_offers": a.second_preference_offers, + "third_preference_offers": a.third_preference_offers, + "cross_la_applications": a.cross_la_applications, + "cross_la_offers": a.cross_la_offers, + } + + +def _census_dict(pc) -> dict: + return { + "year": pc.year, + "total_pupils": pc.total_pupils, + "female_pupils": pc.female_pupils, + "male_pupils": pc.male_pupils, + "fsm_pct": pc.fsm_pct, + "eal_pct": pc.eal_pct, + } + + +def _deprivation_dict(d) -> dict: + return { + "lsoa_code": d.lsoa_code, + "idaci_score": d.idaci_score, + "idaci_decile": d.idaci_decile, + } + + +def _finance_dict(f) -> dict: + return { + "year": f.year, + "per_pupil_spend": f.per_pupil_spend, + "staff_cost_pct": f.staff_cost_pct, + "teacher_cost_pct": f.teacher_cost_pct, + "support_staff_cost_pct": f.support_staff_cost_pct, + "premises_cost_pct": f.premises_cost_pct, + } + + +def _empty_supplementary() -> dict: + return { + "ofsted": None, + "census": None, + "admissions": None, + "admissions_history": [], + "sen_detail": None, + "phonics": None, + "deprivation": None, + "finance": None, + } + + +def get_supplementary_data_batch(db: Session, urns: list[int]) -> dict: + """Fetch supplementary data for many URNs with one query per table + (WHERE urn IN (...)) instead of ~5 queries per school, collapsing the + per-request round-trips from 5*N to a constant 5. Returns {urn: block} + with the same shape get_supplementary_data produces per URN. + + Each table is queried independently and failures degrade that table to + empty for every URN — a missing mart never blanks the others. + """ + urns = [int(u) for u in urns] + result = {urn: _empty_supplementary() for urn in urns} + if not urns: + return result + + def _safe(fn): try: - q = db.query(model).filter(getattr(model, pk_field) == urn) - if latest_field: - q = q.order_by(getattr(model, latest_field).desc()) - return q.first() + fn() except Exception as e: import logging - logging.getLogger(__name__).error("safe_query failed for %s: %s", model.__name__, e) + logging.getLogger(__name__).error("batch supplementary query failed: %s", e) db.rollback() - return None - # Latest Ofsted inspection - o = safe_query(FactOfstedInspection, "urn", "inspection_date") - result["ofsted"] = ( - { - "framework": o.framework, - "inspection_date": o.inspection_date.isoformat() if o.inspection_date else None, - "inspection_type": o.inspection_type, - # Fall back to the grade parsed from an ungraded (Section 8) outcome - # (e.g. "School remains Good") when there's no graded grade, so the - # detail page matches the list badge. - "overall_effectiveness": ( - o.overall_effectiveness - if o.overall_effectiveness is not None - else o.ungraded_grade - ), - "quality_of_education": o.quality_of_education, - "behaviour_attitudes": o.behaviour_attitudes, - "personal_development": o.personal_development, - "leadership_management": o.leadership_management, - "early_years_provision": o.early_years_provision, - "sixth_form_provision": o.sixth_form_provision, - "previous_overall": None, # Not available in new schema - "rc_safeguarding_met": o.rc_safeguarding_met, - "rc_inclusion": o.rc_inclusion, - "rc_curriculum_teaching": o.rc_curriculum_teaching, - "rc_achievement": o.rc_achievement, - "rc_attendance_behaviour": o.rc_attendance_behaviour, - "rc_personal_development": o.rc_personal_development, - "rc_leadership_governance": o.rc_leadership_governance, - "rc_early_years": o.rc_early_years, - "rc_sixth_form": o.rc_sixth_form, - "report_url": o.report_url, - } - if o - else None - ) - - # Census (latest year of fact_pupil_characteristics) - pc = safe_query(FactPupilCharacteristics, "urn", "year") - result["census"] = ( - { - "year": pc.year, - "total_pupils": pc.total_pupils, - "female_pupils": pc.female_pupils, - "male_pupils": pc.male_pupils, - "fsm_pct": pc.fsm_pct, - "eal_pct": pc.eal_pct, - } - if pc - else None - ) - - # Admissions — all years, oldest first (for the multi-year trend view). - def _admissions_row(a): - return { - "year": a.year, - "school_phase": a.school_phase, - "places_offered": a.places_offered, - "total_applications": a.total_applications, - "first_preference_applications": a.first_preference_applications, - "first_preference_offers": a.first_preference_offers, - "first_preference_offer_pct": a.first_preference_offer_pct, - "oversubscription_ratio": a.oversubscription_ratio, - "oversubscribed": a.oversubscribed, - } - - try: - admissions_rows = ( - db.query(FactAdmissions) - .filter(FactAdmissions.urn == urn) - .order_by(FactAdmissions.year.asc()) + # Ofsted — latest inspection per URN. Ordered so the first row seen per + # URN is the most recent. + def _ofsted(): + rows = ( + db.query(FactOfstedInspection) + .filter(FactOfstedInspection.urn.in_(urns)) + .order_by(FactOfstedInspection.urn, FactOfstedInspection.inspection_date.desc()) .all() ) - except Exception as e: - import logging - logging.getLogger(__name__).error("admissions history query failed: %s", e) - db.rollback() - admissions_rows = [] + seen = set() + for o in rows: + if o.urn in seen: + continue + seen.add(o.urn) + result[o.urn]["ofsted"] = _ofsted_block(o, o.urn) + _safe(_ofsted) - history = [_admissions_row(a) for a in admissions_rows] - result["admissions_history"] = history - # Keep the single latest-year object for backwards-compatible consumers - # (hero chips, etc.). - result["admissions"] = history[-1] if history else None + # Census — latest year per URN. + def _census(): + rows = ( + db.query(FactPupilCharacteristics) + .filter(FactPupilCharacteristics.urn.in_(urns)) + .order_by(FactPupilCharacteristics.urn, FactPupilCharacteristics.year.desc()) + .all() + ) + seen = set() + for pc in rows: + if pc.urn in seen: + continue + seen.add(pc.urn) + result[pc.urn]["census"] = _census_dict(pc) + _safe(_census) - # SEN detail — not available in current marts - result["sen_detail"] = None + # Admissions — all years per URN, oldest first (multi-year trend view). + def _admissions(): + rows = ( + db.query(FactAdmissions) + .filter(FactAdmissions.urn.in_(urns)) + .order_by(FactAdmissions.urn, FactAdmissions.year.asc()) + .all() + ) + history: dict = {urn: [] for urn in urns} + for a in rows: + history[a.urn].append(_admissions_row_dict(a)) + for urn, rows_for_urn in history.items(): + result[urn]["admissions_history"] = rows_for_urn + result[urn]["admissions"] = rows_for_urn[-1] if rows_for_urn else None + _safe(_admissions) - # Phonics — no school-level data on EES - result["phonics"] = None + # Deprivation — one row per URN. + def _deprivation(): + rows = ( + db.query(FactDeprivation) + .filter(FactDeprivation.urn.in_(urns)) + .all() + ) + for d in rows: + result[d.urn]["deprivation"] = _deprivation_dict(d) + _safe(_deprivation) - # Deprivation - d = safe_query(FactDeprivation, "urn") - result["deprivation"] = ( - { - "lsoa_code": d.lsoa_code, - "idaci_score": d.idaci_score, - "idaci_decile": d.idaci_decile, - } - if d - else None - ) - - # Finance (latest year) - f = safe_query(FactFinance, "urn", "year") - result["finance"] = ( - { - "year": f.year, - "per_pupil_spend": f.per_pupil_spend, - "staff_cost_pct": f.staff_cost_pct, - "teacher_cost_pct": f.teacher_cost_pct, - "support_staff_cost_pct": f.support_staff_cost_pct, - "premises_cost_pct": f.premises_cost_pct, - } - if f - else None - ) + # Finance — latest year per URN. + def _finance(): + rows = ( + db.query(FactFinance) + .filter(FactFinance.urn.in_(urns)) + .order_by(FactFinance.urn, FactFinance.year.desc()) + .all() + ) + seen = set() + for f in rows: + if f.urn in seen: + continue + seen.add(f.urn) + result[f.urn]["finance"] = _finance_dict(f) + _safe(_finance) return result + + +def get_supplementary_data(db: Session, urn: int) -> dict: + """Supplementary data for a single URN (thin wrapper over the batch).""" + return get_supplementary_data_batch(db, [urn])[int(urn)] diff --git a/backend/models.py b/backend/models.py index 2ab54e3..b2325d9 100644 --- a/backend/models.py +++ b/backend/models.py @@ -88,6 +88,15 @@ class KS2Performance(Base): maths_high_pct = Column(Float) maths_avg_score = Column(Float) maths_progress = Column(Float) + # Progress confidence intervals + writing working-towards (published + # for years with progress measures, i.e. up to 2022/23) + reading_progress_lower_ci = Column(Float) + reading_progress_upper_ci = Column(Float) + writing_progress_lower_ci = Column(Float) + writing_progress_upper_ci = Column(Float) + writing_working_towards_pct = Column(Float) + maths_progress_lower_ci = Column(Float) + maths_progress_upper_ci = Column(Float) gps_expected_pct = Column(Float) gps_high_pct = Column(Float) gps_avg_score = Column(Float) @@ -165,6 +174,11 @@ class FactAdmissions(Base): total_applications = Column(Integer) first_preference_applications = Column(Integer) first_preference_offers = Column(Integer) + total_offers = Column(Integer) + second_preference_offers = Column(Integer) + third_preference_offers = Column(Integer) + cross_la_applications = Column(Integer) + cross_la_offers = Column(Integer) first_preference_offer_pct = Column(Float) oversubscription_ratio = Column(Float) oversubscribed = Column(Boolean) @@ -217,6 +231,23 @@ class FactFinance(Base): premises_cost_pct = Column(Float) +class Ks4NationalAverage(Base): + """Computed national KS4 averages (from our dataset) — one row per year.""" + __tablename__ = "fact_ks4_national_averages" + __table_args__ = MARTS + + year = Column(Integer, primary_key=True) + attainment_8_score = Column(Float) + progress_8_score = Column(Float) + english_maths_standard_pass_pct = Column(Float) + english_maths_strong_pass_pct = Column(Float) + ebacc_entry_pct = Column(Float) + ebacc_standard_pass_pct = Column(Float) + ebacc_strong_pass_pct = Column(Float) + ebacc_avg_score = Column(Float) + gcse_grade_91_pct = Column(Float) + + class Ks2NationalAverage(Base): """Official DfE KS2 national headline averages — one row per academic year.""" __tablename__ = "fact_ks2_national_averages" diff --git a/backend/ofsted_codes.py b/backend/ofsted_codes.py new file mode 100644 index 0000000..81fa229 --- /dev/null +++ b/backend/ofsted_codes.py @@ -0,0 +1,44 @@ +"""Ofsted renewed-framework (Nov 2025) report-card code translation. + +Scale labels are the live-sampled vocabulary from the Ofsted MI file +(see pipeline/scripts/diagnose_compare_gaps.py, TASK 7 VALUE SAMPLE) — +verified against real data, not the consultation draft. +""" + +REPORT_CARD_GRADE_NAMES = { + 1: "Exceptional", + 2: "Strong standard", + 3: "Expected standard", + 4: "Needs attention", + 5: "Urgent improvement", +} + +# Graded evaluation areas only — safeguarding is a separate boolean +# judgement and must never appear in grade counts or label maps. +_RC_AREA_KEYS = ( + "rc_inclusion", + "rc_curriculum_teaching", + "rc_achievement", + "rc_attendance_behaviour", + "rc_personal_development", + "rc_leadership_governance", + "rc_early_years", + "rc_sixth_form", +) + + +def report_card_labels(ofsted: dict) -> dict: + """{area_key: {code, label}} for populated, known-valued rc_* areas.""" + out = {} + for key in _RC_AREA_KEYS: + code = ofsted.get(key) + label = REPORT_CARD_GRADE_NAMES.get(code) + if code is not None and label is not None: + out[key] = {"code": code, "label": label} + return out + + +def ofsted_page_url(urn: int) -> str: + """The school's page on ofsted.gov.uk (all its reports live there — + we never deep-link an individual report).""" + return f"https://reports.ofsted.gov.uk/provider/21/{urn}" diff --git a/backend/tests/test_benchmarks.py b/backend/tests/test_benchmarks.py new file mode 100644 index 0000000..50c8233 --- /dev/null +++ b/backend/tests/test_benchmarks.py @@ -0,0 +1,81 @@ +"""compute_benchmarks: state-school benchmarks computed from our dataset +(spec §5/§8.6). The disadvantaged average must be weighted by cohort size, +medians must ignore NaN, and only the latest year counts.""" + +import numpy as np +import pandas as pd + +from backend.data_loader import compute_benchmarks + +LATEST = 202425 + + +def _df(): + rows = [ + # Six primary schools, latest year. Disadvantaged RWM chosen so the + # weighted average differs clearly from the unweighted mean: + # weighted = (40*100 + 60*300) / 400 = 55.0 ; unweighted mean = 50.0 + dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=100, + rwm_expected_disadvantaged_pct=40.0, eal_pct=10.0, + sen_support_pct=10.0, disadvantaged_pct=20.0, fsm_pct=15.0, total_pupils=200), + dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=300, + rwm_expected_disadvantaged_pct=60.0, eal_pct=20.0, + sen_support_pct=14.0, disadvantaged_pct=24.0, fsm_pct=17.0, total_pupils=280), + dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=np.nan, + rwm_expected_disadvantaged_pct=99.0, eal_pct=30.0, + sen_support_pct=18.0, disadvantaged_pct=30.0, fsm_pct=19.0, total_pupils=300), + dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=50, + rwm_expected_disadvantaged_pct=np.nan, eal_pct=np.nan, + sen_support_pct=np.nan, disadvantaged_pct=np.nan, fsm_pct=np.nan, total_pupils=np.nan), + dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=40, + rwm_expected_disadvantaged_pct=np.nan, eal_pct=40.0, + sen_support_pct=20.0, disadvantaged_pct=40.0, fsm_pct=21.0, total_pupils=350), + dict(year=LATEST, attainment_8_score=np.nan, eligible_pupils=60, + rwm_expected_disadvantaged_pct=np.nan, eal_pct=50.0, + sen_support_pct=22.0, disadvantaged_pct=44.0, fsm_pct=23.0, total_pupils=400), + # Two secondary schools (attainment_8 non-null) + dict(year=LATEST, attainment_8_score=45.0, eligible_pupils=180, + rwm_expected_disadvantaged_pct=np.nan, eal_pct=15.0, + sen_support_pct=12.0, disadvantaged_pct=22.0, fsm_pct=12.0, total_pupils=1000), + dict(year=LATEST, attainment_8_score=50.0, eligible_pupils=200, + rwm_expected_disadvantaged_pct=np.nan, eal_pct=25.0, + sen_support_pct=16.0, disadvantaged_pct=26.0, fsm_pct=14.0, total_pupils=1200), + # An older-year primary row that must NOT influence anything + dict(year=202324, attainment_8_score=np.nan, eligible_pupils=500, + rwm_expected_disadvantaged_pct=1.0, eal_pct=99.0, + sen_support_pct=99.0, disadvantaged_pct=99.0, fsm_pct=99.0, total_pupils=9999), + ] + return pd.DataFrame(rows) + + +def test_weighted_disadvantaged_average(): + b = compute_benchmarks(_df()) + # Row 3 has NaN eligible_pupils and must be excluded from the weighting. + assert b["primary"]["disadvantaged_rwm_expected_pct"] == 55.0 + + +def test_medians_ignore_nan_and_older_years(): + b = compute_benchmarks(_df()) + assert b["year"] == LATEST + # eal medians over [10,20,30,40,50] = 30 + assert b["primary"]["eal_pct"] == 30.0 + # fsm medians over [15,17,19,21,23] = 19 + assert b["primary"]["fsm_pct"] == 19.0 + # median pupils over [200,280,300,350,400] = 300 + assert b["primary"]["median_pupils"] == 300 + + +def test_secondary_block_has_no_disadvantaged_rwm(): + b = compute_benchmarks(_df()) + assert "disadvantaged_rwm_expected_pct" not in b["secondary"] + assert b["secondary"]["fsm_pct"] == 13.0 + assert b["secondary"]["median_pupils"] == 1100 + + +def test_provenance_string(): + b = compute_benchmarks(_df()) + assert b["source"] == "state-school average (computed from our dataset)" + + +def test_empty_df(): + assert compute_benchmarks(pd.DataFrame()) == {} diff --git a/backend/tests/test_compare_enrichment.py b/backend/tests/test_compare_enrichment.py new file mode 100644 index 0000000..0ff671f --- /dev/null +++ b/backend/tests/test_compare_enrichment.py @@ -0,0 +1,123 @@ +"""/api/compare enrichment for the compare redesign: per-school +supplementary blocks, top-level national_averages (shared with the +/api/national-averages endpoint) and computed benchmarks — all additive.""" + +import types + +import numpy as np +import pandas as pd +import pytest +from fastapi.testclient import TestClient + +LATEST = 202425 + +CANNED_SUPPLEMENTARY = { + "ofsted": {"overall_effectiveness": 2, "grade_source": "graded", + "report_card": {}, "ofsted_page_url": "https://reports.ofsted.gov.uk/provider/21/100140"}, + "census": {"year": 202526, "fsm_pct": 29.8}, + "admissions": {"year": 202627, "second_preference_offers": 4}, + "admissions_history": [{"year": 202627, "second_preference_offers": 4}], + "sen_detail": None, + "phonics": None, + "deprivation": {"idaci_decile": 4}, + "finance": None, +} + + +def _two_primary_schools_df() -> pd.DataFrame: + rows = [] + for urn, name, rwm, dis in ((100140, "Plumcroft Primary School", 79.0, 72.0), + (138690, "Barclay Primary School", 87.0, 86.0)): + rows.append(dict( + urn=urn, school_name=name, local_authority="Greenwich", + school_type="Community school", address="1 Road", phase="Primary", + year=LATEST, rwm_expected_pct=rwm, attainment_8_score=np.nan, + eligible_pupils=60, rwm_expected_disadvantaged_pct=dis, + eal_pct=20.0, sen_support_pct=14.0, disadvantaged_pct=25.0, + total_pupils=1000.0, + )) + return pd.DataFrame(rows) + + +class _StubNatRow: + year = 202425 + rwm_expected_pct = 62.1 + gps_expected_pct = 72.0 + science_expected_pct = 81.0 + + +class _StubSession: + def query(self, *a, **k): + return self + + def order_by(self, *a, **k): + return self + + def all(self): + return [_StubNatRow()] + + def close(self): + pass + + +@pytest.fixture() +def client(monkeypatch): + from backend import app as app_module + from backend import database as database_module + + monkeypatch.setattr(app_module, "load_school_data", _two_primary_schools_df) + monkeypatch.setattr( + app_module, + "get_supplementary_data_batch", + lambda db, urns: {int(u): dict(CANNED_SUPPLEMENTARY) for u in urns}, + ) + monkeypatch.setattr(database_module, "SessionLocal", _StubSession) + return TestClient(app_module.app, raise_server_exceptions=False) + + +def test_existing_shape_is_preserved(client): + body = client.get("/api/compare?urns=100140,138690").json() + school = body["comparison"]["100140"] + assert school["school_info"]["rwm_expected_pct"] == 79.0 + assert school["yearly_data"][0]["year"] == LATEST + + +def test_each_school_gains_supplementary_blocks(client): + body = client.get("/api/compare?urns=100140,138690").json() + for urn in ("100140", "138690"): + school = body["comparison"][urn] + assert school["ofsted"]["grade_source"] == "graded" + assert school["census"]["fsm_pct"] == 29.8 + assert school["admissions"]["second_preference_offers"] == 4 + assert school["admissions_history"][0]["year"] == 202627 + assert school["deprivation"]["idaci_decile"] == 4 + + +def test_top_level_national_averages_and_benchmarks(client): + body = client.get("/api/compare?urns=100140,138690").json() + assert body["national_averages"]["year"] == LATEST + assert body["benchmarks"]["source"] == "state-school average (computed from our dataset)" + # weighted over equal cohorts of 72 and 86 = 79.0 + assert body["benchmarks"]["primary"]["disadvantaged_rwm_expected_pct"] == 79.0 + + +def test_supplementary_failure_degrades_not_500(client, monkeypatch): + from backend import app as app_module + + def _boom(db, urns): + raise RuntimeError("marts unavailable") + + monkeypatch.setattr(app_module, "get_supplementary_data_batch", _boom) + resp = client.get("/api/compare?urns=100140") + assert resp.status_code == 200 + school = resp.json()["comparison"]["100140"] + assert school["ofsted"] is None + assert school["admissions_history"] == [] + + +def test_national_averages_endpoint_exposes_gps_science(client): + body = client.get("/api/national-averages").json() + latest_primary_by_year = [e["primary"] for e in body["by_year"] if e["primary"]] + assert latest_primary_by_year, "expected official by_year rows from the stub" + assert latest_primary_by_year[-1]["gps_expected_pct"] == 72.0 + assert latest_primary_by_year[-1]["science_expected_pct"] == 81.0 diff --git a/backend/tests/test_gias_translation.py b/backend/tests/test_gias_translation.py index 7586010..414b8e4 100644 --- a/backend/tests/test_gias_translation.py +++ b/backend/tests/test_gias_translation.py @@ -4,7 +4,7 @@ rest of the backend sees must carry today's name strings.""" import numpy as np import pandas as pd -from backend.data_loader import translate_gias_code_columns +from backend.data_loader import _missing_column_name, translate_gias_code_columns from backend.gias_codes import ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION @@ -42,3 +42,91 @@ def test_missing_code_columns_are_a_noop(): out = translate_gias_code_columns(df) assert out.iloc[0]["phase"] == "Primary" assert out.iloc[0]["status"] == "Open" + + +def _fake_exc(orig_message): + """A stand-in for sqlalchemy.exc.ProgrammingError: str(exc) embeds the + full SQL statement (deliberately containing every column name below, to + prove the matcher doesn't fall back to it), while .orig carries the real + DBAPI error message naming only the offending column.""" + exc = Exception( + "SELECT s.phase_code, s.school_type_code, s.religious_character_code, " + "s.status_code, s.admissions_policy_code, s.has_sixth_form FROM ... " + f"[SQL: ...] (Background on this error at: https://...)" + ) + exc.orig = Exception(orig_message) if orig_message is not None else None + return exc + + +def test_missing_column_name_quoted(): + assert _missing_column_name(_fake_exc('column "phase_code" does not exist')) == "phase_code" + + +def test_missing_column_name_unquoted(): + assert _missing_column_name(_fake_exc("column phase_code does not exist")) == "phase_code" + + +def test_missing_column_name_table_prefixed(): + assert ( + _missing_column_name(_fake_exc("column s.has_sixth_form does not exist")) + == "has_sixth_form" + ) + + +def test_missing_column_name_no_match_returns_none(): + assert _missing_column_name(_fake_exc("relation \"marts.dim_school\" does not exist")) is None + + +def test_load_school_data_survives_premigration_marts(monkeypatch): + """Real prod state until the nightly pipeline first rebuilds the mart with + the GIAS code columns: marts.dim_school still has the old name columns + (phase, school_type, religious_character, status, admissions_policy) + instead of the new *_code columns. The first query raises UndefinedColumn + on s.phase_code; load_school_data_as_dataframe must retry with the + legacy name-column query rather than swallow the error and return (and + then have load_school_data cache) an empty DataFrame.""" + import sqlalchemy.exc + from backend import data_loader + + data_loader._df_cache = None + data_loader._df_latest_cache = None + + good_df = pd.DataFrame( + [ + { + "urn": 1, + "school_name": "Legacy School", + "phase": "Primary", + "school_type": "Academy", + "status": "Open", + } + ] + ) + calls = [] + + def fake_read_sql(query, con): + calls.append(query) + if len(calls) == 1: + raise sqlalchemy.exc.ProgrammingError( + statement=str(data_loader._MAIN_QUERY), + params=None, + orig=Exception( + "(psycopg2.errors.UndefinedColumn) column s.phase_code " + "does not exist\nLINE 5: s.phase_code," + ), + ) + return good_df.copy() + + monkeypatch.setattr(data_loader.pd, "read_sql", fake_read_sql) + + try: + df = data_loader.load_school_data_as_dataframe() + finally: + data_loader._df_cache = None + data_loader._df_latest_cache = None + + assert len(calls) == 2, "must retry with the legacy name-column query variant" + assert calls[1] is data_loader._MAIN_QUERY_LEGACY_NAMES + assert not df.empty + assert df["phase"].iloc[0] == "Primary" + assert df["status"].iloc[0] == "Open" diff --git a/backend/tests/test_national_averages_marts.py b/backend/tests/test_national_averages_marts.py new file mode 100644 index 0000000..b08c6df --- /dev/null +++ b/backend/tests/test_national_averages_marts.py @@ -0,0 +1,93 @@ +"""_national_averages_payload reads persisted marts (computed at import +time) — it must never loop the dataframe per year. The only dataframe work +allowed is the single-latest-year KS4 fallback for the window between a +deploy and the next DAG run.""" + +import numpy as np +import pandas as pd +import pytest + +LATEST = 202425 + + +def _df(): + return pd.DataFrame( + [ + dict(year=202324, attainment_8_score=40.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=50.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=30.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=np.nan, rwm_expected_pct=80.0), + ] + ) + + +class _Ks2Row: + year = LATEST + rwm_expected_pct = 62.1 + gps_expected_pct = 72.0 + + +class _Ks4Row: + year = LATEST + attainment_8_score = 46.5 + progress_8_score = -0.02 + + +class _StubSession: + """Returns KS2 rows for the first query and KS4 rows for the second — + mirroring the payload's query order.""" + + def __init__(self): + self.calls = 0 + + def query(self, model): + self._model = model.__name__ + return self + + def order_by(self, *a): + return self + + def all(self): + return [_Ks2Row()] if self._model == "Ks2NationalAverage" else [_Ks4Row()] + + def close(self): + pass + + +class _Ks4MissingSession(_StubSession): + def all(self): + if self._model == "Ks4NationalAverage": + raise RuntimeError("relation does not exist") + return [_Ks2Row()] + + def rollback(self): + pass + + +@pytest.fixture() +def payload(monkeypatch): + from backend import app as app_module + from backend import database as database_module + + def _run(session_cls): + monkeypatch.setattr(database_module, "SessionLocal", session_cls) + return app_module._national_averages_payload(_df()) + + return _run + + +def test_ks4_averages_come_from_the_mart_not_the_dataframe(payload): + body = payload(_StubSession) + # Mart value (46.5), NOT the dataframe mean of (50+30)/2 = 40.0 + assert body["secondary"]["attainment_8_score"] == 46.5 + assert body["primary"]["rwm_expected_pct"] == 62.1 + assert body["by_year"][-1]["secondary"]["progress_8_score"] == -0.02 + + +def test_missing_ks4_mart_falls_back_to_latest_year_only(payload): + body = payload(_Ks4MissingSession) + # Fallback computes the latest year from the df: mean(50, 30) = 40.0 + assert body["secondary"]["attainment_8_score"] == 40.0 + # ...and only the latest year — no historical KS4 loop + ks4_years = [e["year"] for e in body["by_year"] if e["secondary"]] + assert ks4_years == [LATEST] diff --git a/backend/tests/test_ofsted_codes.py b/backend/tests/test_ofsted_codes.py new file mode 100644 index 0000000..aa8aba4 --- /dev/null +++ b/backend/tests/test_ofsted_codes.py @@ -0,0 +1,45 @@ +"""Report-card code translation uses the live-sampled Ofsted vocabulary +(pipeline/scripts/diagnose_compare_gaps.py, TASK 7 VALUE SAMPLE): +Exceptional / Strong standard / Expected standard / Needs attention / +Urgent improvement — never the consultation draft's 'Attention needed'.""" + +from backend.ofsted_codes import ( + REPORT_CARD_GRADE_NAMES, + ofsted_page_url, + report_card_labels, +) + + +def test_scale_is_sampled_vocabulary(): + assert REPORT_CARD_GRADE_NAMES == { + 1: "Exceptional", + 2: "Strong standard", + 3: "Expected standard", + 4: "Needs attention", + 5: "Urgent improvement", + } + + +def test_labels_only_for_populated_areas_and_never_safeguarding(): + ofsted = { + "rc_achievement": 2, + "rc_inclusion": 3, + "rc_attendance_behaviour": 4, + "rc_early_years": None, + "rc_safeguarding_met": True, + "overall_effectiveness": None, + } + labels = report_card_labels(ofsted) + assert labels == { + "rc_achievement": {"code": 2, "label": "Strong standard"}, + "rc_inclusion": {"code": 3, "label": "Expected standard"}, + "rc_attendance_behaviour": {"code": 4, "label": "Needs attention"}, + } + + +def test_unknown_code_is_skipped_not_crashed(): + assert report_card_labels({"rc_achievement": 9}) == {} + + +def test_provider_url(): + assert ofsted_page_url(138690) == "https://reports.ofsted.gov.uk/provider/21/138690" diff --git a/backend/tests/test_sixth_form_flag.py b/backend/tests/test_sixth_form_flag.py index af32799..a5a87f2 100644 --- a/backend/tests/test_sixth_form_flag.py +++ b/backend/tests/test_sixth_form_flag.py @@ -148,10 +148,14 @@ def test_load_school_data_survives_missing_has_sixth_form_column(monkeypatch): def fake_read_sql(query, con): calls.append(query) if len(calls) == 1: + # The statement text still contains phase_code, school_type_code, + # etc. (it's the full _MAIN_QUERY SELECT list) — that's exactly + # the collision this test guards against: matching must be done + # against exc.orig (the DBAPI error), not str(exc)/the statement. raise sqlalchemy.exc.ProgrammingError( - "SELECT ...", - None, - Exception( + statement=str(data_loader._MAIN_QUERY), + params=None, + orig=Exception( "(psycopg2.errors.UndefinedColumn) column s.has_sixth_form " "does not exist" ), diff --git a/backend/tests/test_supplementary_batch.py b/backend/tests/test_supplementary_batch.py new file mode 100644 index 0000000..0593034 --- /dev/null +++ b/backend/tests/test_supplementary_batch.py @@ -0,0 +1,111 @@ +"""get_supplementary_data_batch fetches one query per table for all URNs +(not ~5 per school) and returns the same per-URN block shape as the +single-URN function, picking the latest row per URN where relevant.""" + +import types + +from backend import data_loader +from backend.data_loader import get_supplementary_data_batch + + +class _FakeQuery: + """Records that a query ran and serves canned rows filtered by an in-list.""" + + def __init__(self, recorder, model_name, rows): + self._rec = recorder + self._model = model_name + self._rows = rows + + def filter(self, *args, **kwargs): + return self + + def order_by(self, *args, **kwargs): + return self + + def all(self): + self._rec.append(self._model) + return self._rows + + def first(self): + self._rec.append(self._model) + return self._rows[0] if self._rows else None + + +class _FakeSession: + def __init__(self, rows_by_model): + self.rows_by_model = rows_by_model + self.queries: list[str] = [] + + def query(self, model): + name = model.__name__ + return _FakeQuery(self.queries, name, self.rows_by_model.get(name, [])) + + def rollback(self): + pass + + +def _ofsted_row(urn, date, oe): + base = {f: None for f in ( + "framework", "inspection_type", "quality_of_education", "behaviour_attitudes", + "personal_development", "leadership_management", "early_years_provision", + "sixth_form_provision", "ungraded_outcome", "ungraded_grade", + "rc_safeguarding_met", "rc_inclusion", "rc_curriculum_teaching", "rc_achievement", + "rc_attendance_behaviour", "rc_personal_development", "rc_leadership_governance", + "rc_early_years", "rc_sixth_form", "report_url", + )} + base.update(urn=urn, inspection_date=types.SimpleNamespace(isoformat=lambda: date), + overall_effectiveness=oe, grade_source=None) + return types.SimpleNamespace(**base) + + +def _adm_row(urn, year): + return types.SimpleNamespace( + urn=urn, year=year, school_phase="Primary", places_offered=100, + total_applications=200, first_preference_applications=150, + first_preference_offers=140, first_preference_offer_pct=93.3, + oversubscription_ratio=1.5, oversubscribed=True, + total_offers=100, second_preference_offers=5, third_preference_offers=2, + cross_la_applications=10, cross_la_offers=3, + ) + + +def test_one_query_per_table_and_latest_row_per_urn(): + rows = { + # URN 1 has two Ofsted rows; the batch must keep the most recent (2023). + "FactOfstedInspection": [ + _ofsted_row(1, "2023-01-01", 2), + _ofsted_row(1, "2019-01-01", 3), + _ofsted_row(2, "2021-06-01", 1), + ], + "FactAdmissions": [_adm_row(1, 202526), _adm_row(1, 202627), _adm_row(2, 202627)], + "FactPupilCharacteristics": [], + "FactDeprivation": [], + "FactFinance": [], + } + session = _FakeSession(rows) + out = get_supplementary_data_batch(session, [1, 2]) + + # Exactly one query per table — five total, regardless of two URNs. + assert sorted(session.queries) == [ + "FactAdmissions", "FactDeprivation", "FactFinance", + "FactOfstedInspection", "FactPupilCharacteristics", + ] + + # Latest Ofsted kept per URN + assert out[1]["ofsted"]["overall_effectiveness"] == 2 + assert out[2]["ofsted"]["overall_effectiveness"] == 1 + + # Admissions history grouped per URN, latest exposed as `admissions` + assert [r["year"] for r in out[1]["admissions_history"]] == [202526, 202627] + assert out[1]["admissions"]["year"] == 202627 + assert out[2]["admissions_history"] == [{**out[2]["admissions_history"][0]}] + + # Empty tables degrade to the null block, not a crash + assert out[1]["census"] is None and out[1]["deprivation"] is None + + +def test_single_wrapper_matches_batch(monkeypatch): + session = _FakeSession({"FactOfstedInspection": [_ofsted_row(5, "2022-01-01", 2)]}) + single = data_loader.get_supplementary_data(session, 5) + assert single["ofsted"]["overall_effectiveness"] == 2 + assert single["admissions_history"] == [] diff --git a/backend/tests/test_supplementary_enrichment.py b/backend/tests/test_supplementary_enrichment.py new file mode 100644 index 0000000..fe6e746 --- /dev/null +++ b/backend/tests/test_supplementary_enrichment.py @@ -0,0 +1,65 @@ +"""Supplementary-block enrichment for the compare redesign: report-card +labels, provider-page URL, graded-vs-carried-forward provenance, and the +admissions preference/cross-LA detail promoted in the data-foundation PR.""" + +import types + +from backend.data_loader import _admissions_row_dict, _ofsted_block + + +def _row(**kw): + base = dict( + framework="RC", inspection_date=None, inspection_type=None, + overall_effectiveness=None, quality_of_education=None, + behaviour_attitudes=None, personal_development=None, + leadership_management=None, early_years_provision=None, + sixth_form_provision=None, ungraded_outcome=None, ungraded_grade=None, + rc_safeguarding_met=None, rc_inclusion=None, rc_curriculum_teaching=None, + rc_achievement=None, rc_attendance_behaviour=None, + rc_personal_development=None, rc_leadership_governance=None, + rc_early_years=None, rc_sixth_form=None, report_url=None, + ) + base.update(kw) + return types.SimpleNamespace(**base) + + +def test_report_card_block_and_provider_url(): + o = _row(rc_achievement=2, rc_inclusion=3, rc_safeguarding_met=True) + block = _ofsted_block(o, urn=100140) + assert block["report_card"]["rc_achievement"]["label"] == "Strong standard" + assert "rc_safeguarding_met" not in block["report_card"] + assert block["rc_safeguarding_met"] is True + assert block["ofsted_page_url"] == "https://reports.ofsted.gov.uk/provider/21/100140" + + +def test_grade_source_graded_vs_carried_forward(): + assert _ofsted_block(_row(overall_effectiveness=1), urn=1)["grade_source"] == "graded" + carried = _ofsted_block(_row(ungraded_grade=2), urn=1) + assert carried["grade_source"] == "ungraded_carried_forward" + assert carried["overall_effectiveness"] == 2 + assert _ofsted_block(_row(), urn=1)["grade_source"] is None + + +def test_ofsted_block_keeps_existing_keys(): + block = _ofsted_block(_row(overall_effectiveness=2, quality_of_education=2), urn=1) + for key in ("framework", "inspection_date", "overall_effectiveness", + "quality_of_education", "rc_inclusion", "report_url"): + assert key in block + + +def test_admissions_row_new_fields(): + a = types.SimpleNamespace( + year=202627, school_phase="Primary", places_offered=80, + total_applications=185, first_preference_applications=74, + first_preference_offers=74, first_preference_offer_pct=100.0, + oversubscription_ratio=0.925, oversubscribed=False, + total_offers=80, second_preference_offers=4, third_preference_offers=2, + cross_la_applications=12, cross_la_offers=3, + ) + d = _admissions_row_dict(a) + for k in ("total_offers", "second_preference_offers", "third_preference_offers", + "cross_la_applications", "cross_la_offers"): + assert d[k] == getattr(a, k) + # Existing keys unchanged + assert d["first_preference_offer_pct"] == 100.0 + assert d["oversubscribed"] is False diff --git a/claude.md b/claude.md index e8555ad..3d05e66 100644 --- a/claude.md +++ b/claude.md @@ -112,11 +112,15 @@ Full details in `docs/DEPLOY.md`. The short version: - **Never push to `main` directly.** Work on a feature branch and open a PR; branch protection requires the PR checks (typecheck, tests, builds, AI review) to pass before merge. -- Merging to `main` deploys automatically: images are built once, deployed to - the **staging** Portainer stack, verified by the Playwright journeys in - `e2e/`, and only then retagged `:prod` and rolled out to production. +- Merging to `main` deploys automatically **to staging only**: images are + built once, deployed to the staging Portainer stack, and verified by the + Playwright journeys in `e2e/`. Production is a second, manual approval: + the "Promote to Production (manual)" workflow in Gitea Actions, run after + testing the feature on staging. It refuses commits whose staging E2E gate + isn't green. Never trigger it yourself — promotion is the human's call. - If you change user-facing behaviour, update or extend the `e2e/` journey - tests in the same PR — they are the promotion gate. + tests in the same PR — they gate whether staging is fit for human testing + and whether a commit is promotable. ## Recent Changes diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 1088fac..02123f6 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -1,41 +1,61 @@ # SDLC & Deployment Pipeline -SchoolCompare uses a fully automated staging → production pipeline on Gitea -Actions. AI writes the code on feature branches; the pipeline verifies every -change on a staging environment before promoting the exact same images to -production. Human input is directional only: feature requests, PR review if -desired, and intervention when a gate fails. +SchoolCompare uses a two-stage deploy model on Gitea Actions with two human +approvals. AI writes the code on feature branches; the first approval merges +the PR, which deploys to staging and runs the E2E gate; the second approval — +after manual testing on staging — promotes the exact same images to +production via a manual workflow. ## The flow ``` feature branch (AI-authored) - │ PR to main + │ PR to main ← approval #1 ▼ PR checks (.gitea/workflows/pr-checks.yml) typecheck + unit tests + backend smoke + image builds (no push) + Claude code review posted as a PR comment (severe findings fail the check) │ merge (branch protection requires green checks) ▼ -Deploy pipeline (.gitea/workflows/deploy.yml) +Stage pipeline (.gitea/workflows/deploy.yml) — automatic 1. build & push images → tags sha-, staging 2. staging Portainer webhook → wait for staging health - 3. Playwright E2E journeys against staging - 4. retag sha- → :prod (same bytes — build once, promote the image) + 3. Playwright E2E journeys against staging ← gate before human testing + ▼ +Manual testing on staging (stx.schoolcompare.co.uk) + │ Actions → "Promote to Production (manual)" ← approval #2 + ▼ +Promote pipeline (.gitea/workflows/promote.yml) — manual dispatch + 1. resolve target sha (input, or latest main if empty) + 2. REFUSE unless that commit's "E2E Journeys against Staging" status is green + 3. retag sha- → :prod (same bytes — build once, promote the image) previous :prod saved as :prod-previous - 5. prod Portainer webhook → wait for prod health + 4. prod Portainer webhook → wait for prod health ``` Key principle: **build once, promote the exact image**. Production pins `:prod`, -which only moves after the E2E gate passes on staging. Nothing tags `:latest` +which only moves when a human runs the promote workflow — and the workflow +only accepts commits that passed the staging E2E gate. Nothing tags `:latest` anymore. ## Branch & PR workflow - `main` is protected: no direct pushes, PRs require green status checks. - All work (human or AI) happens on feature branches → PR to `main`. -- Merging to `main` **is** the release action. If staging or the E2E gate - fails, production is untouched. +- Merging to `main` releases **to staging only**. Production moves only on + the second approval. If staging or the E2E gate fails, fix forward — + production is untouched either way. + +## Promotion granularity + +Staging always runs the latest `main`. Promoting approves a *state of main*, +not a single PR — if two PRs merged since the last promotion, they ship +together. Test staging accordingly. To promote an older state, pass its +commit SHA to the promote workflow (its images must still exist in the +registry). + +Staging quirk for manual testing: external `/api` is broken at the staging +proxy — exercise API endpoints from the host, not via the public staging URL. ## Environments @@ -92,8 +112,9 @@ fail the E2E gate. That's the point: staging absorbs the risk. ## Rollback -Every promotion first re-points `:prod-previous` at the outgoing `:prod`. -To roll back: +Re-run "Promote to Production (manual)" with the SHA of the last good commit +(fastest, fully gated), or manually re-point the tags — every promotion first +saves the outgoing `:prod` as `:prod-previous`: ```bash for img in backend frontend pipeline; do diff --git a/docs/superpowers/plans/2026-07-12-compare-data-foundation.md b/docs/superpowers/plans/2026-07-12-compare-data-foundation.md new file mode 100644 index 0000000..30a3bb8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-compare-data-foundation.md @@ -0,0 +1,591 @@ +# Compare-Screen Data Foundation (Pipeline PR) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Land every pipeline/dbt change the compare-screen redesign needs (spec §5 + §8 of `docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md`): promote raw-but-unstored fields to marts, close the national-averages gaps, and wire the Ofsted report-card columns. + +**Architecture:** Meltano Singer taps load `raw.*` tables; dbt builds `staging` → `marts` (read-only for the backend). All changes here are additive columns/rows — no breaking changes to existing marts. The full `dbt build` runs on the server via the Airflow DAGs; locally we gate with `dbt parse` (no DB needed) plus network-only diagnostic scripts. + +**Tech Stack:** Python (Singer SDK taps), dbt-postgres ~1.10 (invoked as `python -m dbt.cli.main`), Meltano, PostgreSQL. + +## Global Constraints + +- **No new external sources** (spec §5): only fields already in the `raw` schema or in files the taps already download. The one sanctioned tap change is the Ofsted MI report-card columns (spec §5, §8.4) and the legacy-KS2 year addition (same DfE performance-tables source). +- **Additive only:** never rename or drop existing mart columns; the backend maps them 1:1 in `backend/models.py`. +- **Never push to `main`.** Branch: `feat/compare-data-foundation`; PR checks must pass. +- Backend `models.py` changes belong to the follow-up backend PR, not this one. +- dbt invocation is always `python -m dbt.cli.main` (a bare `dbt` resolves to the wrong binary — see `pipeline/dags/school_data_pipeline.py:27`). +- EES suppression codes `z`/`c`/`x` must go through the `safe_numeric` macro. +- Computed benchmarks (FSM/EAL/SEN medians, disadvantaged national average) are **backend work** (spec §5) — explicitly out of scope here. + +--- + +### Task 0: Create the branch + +**Files:** none + +- [ ] **Step 1:** `git checkout main && git pull && git checkout -b feat/compare-data-foundation` + +--- + +### Task 1: Diagnostics — pin the three unknowns + +The spec flags three facts we must confirm from the actual files before wiring code: (a) why `gps_expected_pct`/`science_expected_pct` are NULL in `marts.fact_ks2_national_averages` despite being mapped end-to-end; (b) what the KS2 attainment long file calls its subjects/years for 2021/22 and 2022/23 (subject-level 2022/23 is NULL in prod; school-level 2021/22 is absent); (c) the exact report-card column headers in the current Ofsted MI CSV. + +**Files:** +- Create: `pipeline/scripts/diagnose_compare_gaps.py` + +**Interfaces:** +- Produces: a printed findings report; Tasks 5, 6, 7 consume the confirmed column/label names. Precedent: `pipeline/scripts/diagnose_ees_ks4.py`. + +- [ ] **Step 1: Write the diagnostic script** + +```python +"""Diagnose the three data gaps blocking the compare-screen redesign. + +Run from repo root (network access required, no DB needed): + python pipeline/scripts/diagnose_compare_gaps.py +""" +import io +import re +import sys +import zipfile + +import pandas as pd +import requests + +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees") +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted") +from tap_uk_ees.tap import ( # noqa: E402 + _KS2_NATIONAL_COL_MAP, + _KS2_NATIONAL_CSV_URL, + download_release_zip, + get_all_releases, +) +from tap_uk_ofsted.tap import discover_csv_url # noqa: E402 + +TIMEOUT = 120 + + +def check_national_gps_science(): + print("\n=== (a) National catalogue CSV: GPS/science columns ===") + resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False) + df.columns = [c.strip().lower() for c in df.columns] + for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"): + status = "PRESENT" if csv_col in df.columns else "MISSING" + print(f" {csv_col}: {status}") + gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c] + print(f" all gps/science-ish columns: {gps_like}") + nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"] + print(f" national rows time_periods: {sorted(nat['time_period'].unique())}") + # Sample the values our map would read for the latest year + latest = nat[nat["time_period"] == nat["time_period"].max()] + for csv_col, field in _KS2_NATIONAL_COL_MAP.items(): + val = latest.iloc[0].get(csv_col, "") if len(latest) else "" + print(f" {field} <- {csv_col} = {val!r}") + + +def check_ks2_attainment_years_subjects(): + print("\n=== (b) EES KS2 attainment: years & subject labels ===") + releases = get_all_releases("key-stage-2-attainment") + print(f" releases found: {[r['time_period'] for r in releases]}") + for release in releases: + zf = download_release_zip(release["id"]) + name = next((n for n in zf.namelist() + if "ks2_school_attainment_data" in n and n.endswith(".csv")), None) + if not name: + print(f" {release['time_period']}: NO school attainment CSV in ZIP") + continue + with zf.open(name) as f: + df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000) + years = sorted(df["time_period"].unique()) + subjects = sorted(df["subject"].unique()) + print(f" release {release['time_period']}: time_periods={years}") + print(f" subjects={subjects}") + + +def check_ofsted_report_card_columns(): + print("\n=== (c) Ofsted MI CSV: report-card columns ===") + url = discover_csv_url() + print(f" MI file: {url}") + resp = requests.get(url, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5) + rc_like = [c for c in df.columns + if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)] + print(f" candidate report-card columns ({len(rc_like)}):") + for c in rc_like: + print(f" - {c!r}") + + +if __name__ == "__main__": + check_national_gps_science() + check_ks2_attainment_years_subjects() + check_ofsted_report_card_columns() +``` + +Note: if `_KS2_NATIONAL_CSV_URL` is named differently in `tap_uk_ees/tap.py` (it is defined near the `_KS2_NATIONAL_COL_MAP` around line ~490), import whatever constant holds the catalogue CSV URL. + +- [ ] **Step 2: Run it and record findings** + +Run: `python pipeline/scripts/diagnose_compare_gaps.py 2>&1 | tee /tmp/compare-gaps-findings.txt` +Expected: three sections printed. Paste the findings as a comment block at the bottom of the script (so they're committed evidence), e.g. `# FINDINGS 2026-07-12: pt_gps_exp MISSING (actual col: ...), 202122 present in release X, rc columns: [...]`. + +- [ ] **Step 3: Commit** + +```bash +git add pipeline/scripts/diagnose_compare_gaps.py +git commit -m "chore(pipeline): diagnostic for compare-screen data gaps" +``` + +--- + +### Task 2: Admissions preference detail → mart + +Staging already extracts `second_preference_offers`, `third_preference_offers`, `total_offers` (`stg_ees_admissions.sql:26-29`) — the mart drops them. The cross-LA fields are declared in the tap (`all_applications_from_another_LA`, `offers_to_applicants_from_another_LA`) but not selected in staging. + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_admissions.sql` (after line 33, in `renamed`) +- Modify: `pipeline/transform/models/marts/fact_admissions.sql` +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_admissions block, ~line 120) + +**Interfaces:** +- Produces mart columns: `total_offers int`, `second_preference_offers int`, `third_preference_offers int`, `cross_la_applications int`, `cross_la_offers int`. The backend PR will map these in `FactAdmissions`. + +- [ ] **Step 1: Add cross-LA columns to staging** + +In `stg_ees_admissions.sql`, after the `first_preference_applications` line (line 33): + +```sql + -- Cross-borough demand: applications naming this school from families + -- living in another local authority, and offers made to them. + {{ safe_numeric('"all_applications_from_another_LA"') }}::integer as cross_la_applications, + {{ safe_numeric('"offers_to_applicants_from_another_LA"') }}::integer as cross_la_offers, +``` + +(Quote the identifiers — the tap emits them with mixed case, same trap as `FSM_eligible_percent`, see the header comment in that file. If `dbt parse` or the DAG run later shows the raw columns are lower-cased in Postgres, drop the double quotes.) + +- [ ] **Step 2: Pass everything through the mart** + +Replace the full select list in `fact_admissions.sql`: + +```sql +-- Mart: School admissions — one row per URN per year + +select + urn, + year, + school_phase, + places_offered, + total_offers, + total_applications, + first_preference_applications, + first_preference_offers, + second_preference_offers, + third_preference_offers, + cross_la_applications, + cross_la_offers, + first_preference_offer_pct, + oversubscription_ratio, + oversubscribed, + admissions_policy +from {{ ref('stg_ees_admissions') }} +``` + +- [ ] **Step 3: Add schema tests** + +In `_marts_schema.yml` under `fact_admissions.columns`, append: + +```yaml + - name: second_preference_offers + - name: third_preference_offers + - name: cross_la_applications + - name: cross_la_offers + - name: total_offers +``` + +- [ ] **Step 4: Parse gate** + +Run: `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` +Expected: `Done.` with no compilation errors. + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_admissions.sql pipeline/transform/models/marts/fact_admissions.sql pipeline/transform/models/marts/_marts_schema.yml +git commit -m "feat(pipeline): admissions preference breakdown and cross-LA demand in marts" +``` + +--- + +### Task 3: KS2 progress confidence intervals + writing working-towards + +The tap already emits `progress_measure_lower_conf_interval`, `progress_measure_upper_conf_interval`, `working_towards_expected_standard_pupil_percent` (tap.py:203-206). The staging pivot drops them. These power the CI-based Above/Average/Below progress chips (spec §8, first-review item on statistical honesty). + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_ks2.sql` (inside the `pivoted` CTE, next to each subject's `progress_measure_score` case, lines ~41/55/72, and in the final select ~lines 145-152) +- Modify: `pipeline/transform/models/marts/fact_ks2_performance.sql` +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_ks2_performance block, ~line 82) + +**Interfaces:** +- Produces mart columns: `reading_progress_lower_ci`, `reading_progress_upper_ci`, `writing_progress_lower_ci`, `writing_progress_upper_ci`, `maths_progress_lower_ci`, `maths_progress_upper_ci` (float), `writing_working_towards_pct` (float). + +- [ ] **Step 1: Add pivot cases in staging** + +After the `reading_progress` case (line ~41), add: + +```sql + max(case when subject = 'Reading' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as reading_progress_lower_ci, + max(case when subject = 'Reading' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as reading_progress_upper_ci, +``` + +After the `writing_progress` case (line ~55), add: + +```sql + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as writing_progress_lower_ci, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as writing_progress_upper_ci, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('working_towards_expected_standard_pupil_percent') }} end) as writing_working_towards_pct, +``` + +After the `maths_progress` case (line ~72), add: + +```sql + max(case when subject = 'Maths' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as maths_progress_lower_ci, + max(case when subject = 'Maths' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as maths_progress_upper_ci, +``` + +Then add the seven new columns to the model's final select (next to the existing `p.reading_progress` / `p.writing_progress` / `p.maths_progress` lines ~145-152): + +```sql + p.reading_progress_lower_ci, + p.reading_progress_upper_ci, + p.writing_progress_lower_ci, + p.writing_progress_upper_ci, + p.writing_working_towards_pct, + p.maths_progress_lower_ci, + p.maths_progress_upper_ci, +``` + +- [ ] **Step 2: Pass through the mart** + +In `fact_ks2_performance.sql`, add the same seven column names to the select list immediately after the existing `maths_progress` line (this mart selects staging columns by name; match the file's existing alias style — if columns are selected bare, add them bare). + +- [ ] **Step 3: Schema tests** + +In `_marts_schema.yml` under `fact_ks2_performance.columns`, append the seven names (no tests beyond presence — values are legitimately NULL for 2023/24+ since progress measures ended with 2022/23, spec §4.3): + +```yaml + - name: reading_progress_lower_ci + - name: reading_progress_upper_ci + - name: writing_progress_lower_ci + - name: writing_progress_upper_ci + - name: writing_working_towards_pct + - name: maths_progress_lower_ci + - name: maths_progress_upper_ci +``` + +- [ ] **Step 4: Parse gate** + +Run: `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` +Expected: `Done.` + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_ks2.sql pipeline/transform/models/marts/fact_ks2_performance.sql pipeline/transform/models/marts/_marts_schema.yml +git commit -m "feat(pipeline): KS2 progress confidence intervals and writing working-towards" +``` + +--- + +### Task 4: KS4 — Progress 8 banding and disadvantage gaps + +The tap's `ees_ks4_info` stream already declares `progress8_banding` (DfE's own "well above average … well below average" label — the ready-made secondary chip), `attainment8_diffn` and `progress8_diffn` (tap.py:338-340). Wire them through staging into the mart. + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_ks4.sql` (the CTE that reads `ees_ks4_info` — the same one that already surfaces `sen_pct`; add three columns to its select and to the final joined select) +- Modify: `pipeline/transform/models/marts/fact_ks4_performance.sql` (add after `progress_8_upper_ci`) +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_ks4_performance block, ~line 93) + +**Interfaces:** +- Produces mart columns: `progress_8_banding text`, `attainment_8_disadvantage_gap float`, `progress_8_disadvantage_gap float`. + +- [ ] **Step 1: Staging — select from the info source** + +In the info CTE of `stg_ees_ks4.sql` add: + +```sql + nullif(trim(progress8_banding), '') as progress_8_banding, + {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap, + {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap, +``` + +and add the three names to the model's final select (aliased the same way the CTE's other columns are). + +- [ ] **Step 2: Mart passthrough** + +In `fact_ks4_performance.sql`, after the `progress_8_upper_ci,` line: + +```sql + progress_8_banding, + attainment_8_disadvantage_gap, + progress_8_disadvantage_gap, +``` + +- [ ] **Step 3: Schema tests** — append the three names under `fact_ks4_performance.columns`, plus an accepted-values guard that tolerates NULL: + +```yaml + - name: progress_8_banding + tests: + - accepted_values: + values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average'] + config: + where: "progress_8_banding is not null" + - name: attainment_8_disadvantage_gap + - name: progress_8_disadvantage_gap +``` + +(If the DAG run later shows different capitalisation in the data, fix the accepted values to match the data, not vice versa.) + +- [ ] **Step 4: Parse gate** — `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` → `Done.` + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_ks4.sql pipeline/transform/models/marts/fact_ks4_performance.sql pipeline/transform/models/marts/_marts_schema.yml +git commit -m "feat(pipeline): Progress 8 banding and KS4 disadvantage gaps in marts" +``` + +--- + +### Task 5: National averages — 2015/16 row and GPS/science/scaled-score fix + +Two changes. (1) `stg_ees_ks2_national.sql:34` filters `>= 201617`, which is exactly why the England line starts a year late (2015/16 RWM = 53% exists in the catalogue). (2) GPS/science expected are NULL in prod despite full end-to-end mapping — Task 1's findings say whether the catalogue CSV column names differ from `_KS2_NATIONAL_COL_MAP` (`pt_gps_exp`, `pt_scita_exp`) or whether values are suppressed at source. + +**Files:** +- Modify: `pipeline/transform/models/staging/stg_ees_ks2_national.sql:34` +- Modify (conditional on Task 1 findings): `pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py` (`_KS2_NATIONAL_COL_MAP`) + +**Interfaces:** +- Produces: a 201516 row in `marts.fact_ks2_national_averages`; non-NULL `gps_expected_pct`, `science_expected_pct`, `reading_avg_score`, `maths_avg_score`, `gps_avg_score` for years the DfE publishes them. Backend/frontend consume via `/api/national-averages` unchanged (additive year + newly non-NULL fields). + +- [ ] **Step 1: Widen the year filter** + +In `stg_ees_ks2_national.sql`, change line 34: + +```sql + and cast(trim(time_period) as integer) >= 201516 +``` + +(2015/16 was the first year of the current expected-standard tests; nothing earlier is comparable, so keep a floor.) + +- [ ] **Step 2: Fix the column map per Task 1 findings** + +If Task 1 reported the actual CSV column names for GPS/science/scaled scores differ, update `_KS2_NATIONAL_COL_MAP` in `tap.py` accordingly, e.g. (illustrative — use the diagnosed names): + +```python +_KS2_NATIONAL_COL_MAP = { + # ... existing entries ... + "pt_gps_exp": "gps_expected_pct", # replace key with diagnosed name + "pt_scita_exp": "science_expected_pct", # replace key with diagnosed name +} +``` + +If Task 1 showed the columns are present but suppressed (`x`) at national level for all years, instead delete the two entries from the map, delete the corresponding lines from `stg_ees_ks2_national.sql` and `fact_ks2_national_averages.sql`, and record in the PR description that GPS/science England ticks stay "not in dataset" (the mockups already carry that caveat). + +- [ ] **Step 3: Parse gate** — `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` → `Done.` + +- [ ] **Step 4: Commit** + +```bash +git add pipeline/transform/models/staging/stg_ees_ks2_national.sql pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py +git commit -m "fix(pipeline): include 2015/16 national averages; fix GPS/science national mapping" +``` + +--- + +### Task 6: Legacy KS2 — load the 2021/22 school-level year + +School-level 2021/22 exists in DfE performance-tables archives (same source as the four legacy years already loaded) but in neither our legacy config (stops at 201819, `pipeline/meltano.yml:33-37`) nor EES (starts 2022/23) — unless Task 1's finding (b) showed an EES release carrying 202122, in which case skip this task and note why in the PR. + +The legacy URLs point at the self-hosted filebrowser (`10.0.1.224:8081`) — **the 2021/22 DfE archive must be uploaded there first; this is the one human dependency in this plan.** + +**Files:** +- Modify: `pipeline/meltano.yml` (legacy_ks2_urls block, line ~33) + +**Interfaces:** +- Produces: `raw.legacy_ks2` rows with `year = '202122'`, flowing through `stg_legacy_ks2` → `fact_ks2_performance` unchanged (the stream maps old column names already; 2021/22 CSVs use the same `PTRWM_EXP`-style headers as 2018/19). + +- [ ] **Step 1: Verify the 2021/22 CSV headers match `_LEGACY_KS2_COLUMN_MAP`** + +Download the DfE 2021/22 KS2 revised archive (gov.uk "Compare School Performance data download": 2021-2022 all-schools ZIP), then: + +Run: `python -c "import zipfile,io,pandas as pd; zf=zipfile.ZipFile('/path/to/2021-2022.zip'); n=[x for x in zf.namelist() if 'ks2final' in x.lower() and x.endswith('.csv')][0]; df=pd.read_csv(zf.open(n), dtype=str, nrows=5); import sys; sys.path.insert(0,'pipeline/plugins/extractors/tap-uk-ees'); from tap_uk_ees.tap import _LEGACY_KS2_COLUMN_MAP as m; missing=[c for c in m if c not in df.columns]; print('missing legacy columns:', missing)"` +Expected: `missing legacy columns: []` (progress columns `READPROG` etc. may legitimately be missing/blank in 2021/22 — acceptable, they load as NULL). + +- [ ] **Step 2: Upload the archive to the filebrowser and add the config entry** + +In `pipeline/meltano.yml` under `legacy_ks2_urls`, add (with the real share URL from the filebrowser upload): + +```yaml + "202122": "http://10.0.1.224:8081/filebrowser/api/public/dl/?inline=true" +``` + +- [ ] **Step 3: Commit** + +```bash +git add pipeline/meltano.yml +git commit -m "feat(pipeline): load 2021/22 school-level KS2 from legacy performance tables" +``` + +- [ ] **Step 4 (only if Task 1(b) showed 2022/23 subject labels differ):** widen the subject matchers in `stg_ees_ks2.sql` the same way GPS already is (`subject ilike '%grammar%' or subject = 'GPS'`), e.g. `subject in ('Reading', 'reading')` → use the diagnosed labels. Parse-gate and commit as `fix(pipeline): match 2022/23 KS2 subject labels`. + +--- + +### Task 7: Ofsted report-card columns (rc_*) + +Resolves the tap TODO (`stg_ofsted_inspections.sql:37`). The marts/backed columns already exist as stubs; this wires real values. Uses Task 1(c)'s confirmed MI column names — the candidates below follow the MI file's existing naming style and must be corrected against the diagnostic output. + +**Files:** +- Modify: `pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py` (COLUMN_PRIORITY ~line 19-72, schema ~line 100-114) +- Create: `pipeline/transform/macros/parse_report_card_grade.sql` +- Modify: `pipeline/transform/models/staging/stg_ofsted_inspections.sql:36-46` + +**Interfaces:** +- Produces mart columns (already declared in `fact_ofsted_inspection`): `rc_safeguarding_met boolean`, and `rc_inclusion` … `rc_sixth_form` as integers on the 5-point scale `1=Exceptional, 2=Strong standard, 3=Expected standard, 4=Needs attention/Attention needed, 5=Urgent improvement`. The backend translates codes to labels (same pattern as `gias_codes.py`), verifying wording against Ofsted's published toolkit (spec §8.4). + +- [ ] **Step 1: Add tap column mappings** + +In `COLUMN_PRIORITY` add (replace candidate strings with Task 1(c)'s exact headers — keep them as priority lists so older files degrade to blank): + +```python + "rc_safeguarding_met": ["Report card safeguarding", "Safeguarding"], + "rc_inclusion": ["Report card inclusion", "Inclusion"], + "rc_curriculum_teaching": ["Report card curriculum and teaching", "Curriculum and teaching"], + "rc_achievement": ["Report card achievement", "Achievement"], + "rc_attendance_behaviour": ["Report card attendance and behaviour", "Attendance and behaviour"], + "rc_personal_development": ["Report card personal development and well-being", "Personal development and well-being"], + "rc_leadership_governance": ["Report card leadership and governance", "Leadership and governance"], + "rc_early_years": ["Report card early years", "Early years"], + "rc_sixth_form": ["Report card sixth form", "Sixth form"], +``` + +And in the stream schema (next to `report_url`, ~line 114): + +```python + th.Property("rc_safeguarding_met", th.StringType), + th.Property("rc_inclusion", th.StringType), + th.Property("rc_curriculum_teaching", th.StringType), + th.Property("rc_achievement", th.StringType), + th.Property("rc_attendance_behaviour", th.StringType), + th.Property("rc_personal_development", th.StringType), + th.Property("rc_leadership_governance", th.StringType), + th.Property("rc_early_years", th.StringType), + th.Property("rc_sixth_form", th.StringType), +``` + +- [ ] **Step 2: Write the grade-parsing macro** + +`pipeline/transform/macros/parse_report_card_grade.sql`: + +```sql +{% macro parse_report_card_grade(column_name) %} + case lower(trim(nullif({{ column_name }}, 'NULL'))) + when 'exceptional' then 1 + when 'strong standard' then 2 + when 'expected standard' then 3 + when 'needs attention' then 4 + when 'attention needed' then 4 + when 'urgent improvement' then 5 + end +{% endmacro %} +``` + +- [ ] **Step 3: Wire staging** + +Replace `stg_ofsted_inspections.sql` lines 36-46 (the NULL stubs) with: + +```sql + -- Report Card fields (post-Nov 2025 framework), 5-point scale: + -- 1 Exceptional · 2 Strong standard · 3 Expected standard + -- · 4 Needs attention · 5 Urgent improvement + (lower(trim(nullif(rc_safeguarding_met, 'NULL'))) = 'met') as rc_safeguarding_met, + {{ parse_report_card_grade('rc_inclusion') }}::integer as rc_inclusion, + {{ parse_report_card_grade('rc_curriculum_teaching') }}::integer as rc_curriculum_teaching, + {{ parse_report_card_grade('rc_achievement') }}::integer as rc_achievement, + {{ parse_report_card_grade('rc_attendance_behaviour') }}::integer as rc_attendance_behaviour, + {{ parse_report_card_grade('rc_personal_development') }}::integer as rc_personal_development, + {{ parse_report_card_grade('rc_leadership_governance') }}::integer as rc_leadership_governance, + {{ parse_report_card_grade('rc_early_years') }}::integer as rc_early_years, + {{ parse_report_card_grade('rc_sixth_form') }}::integer as rc_sixth_form, +``` + +Note `rc_safeguarding_met` becomes boolean (NULL when blank) — matching `fact_ofsted_inspection`'s `rc_safeguarding_met` Boolean column. If `fact_ofsted_inspection.sql` casts these columns, align its casts too (inspect that model; it currently passes the text stubs through). + +- [ ] **Step 4: Parse gate + tap smoke test** + +Run: `cd pipeline/transform && python -m dbt.cli.main parse --profiles-dir .` → `Done.` +Run: `python -c "import sys; sys.path.insert(0,'pipeline/plugins/extractors/tap-uk-ofsted'); from tap_uk_ofsted.tap import COLUMN_PRIORITY; assert 'rc_inclusion' in COLUMN_PRIORITY; print('ok')"` → `ok` + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py pipeline/transform/macros/parse_report_card_grade.sql pipeline/transform/models/staging/stg_ofsted_inspections.sql +git commit -m "feat(pipeline): extract Ofsted report-card judgements (rc_* columns)" +``` + +--- + +### Task 8: PR + post-merge verification + +**Files:** none new + +- [ ] **Step 1: Push and open the PR** (Gitea — use the git credential helper + basic-auth API pattern; token-header auth 401s): + +```bash +git push -u origin feat/compare-data-foundation +# then create the PR via the Gitea API with basic auth from `git credential fill` +``` + +PR body: link spec §5/§8, list the new mart columns, note the Task 6 human dependency (filebrowser upload) and the Task 1 findings file. + +- [ ] **Step 2: After merge, verify the DAG run picked everything up** + +The daily/monthly DAGs rebuild the affected models (`pipeline/dags/school_data_pipeline.py`). Spot-check via the public API (production after promotion, staging first at stx.schoolcompare.co.uk — note external /api is broken at the staging proxy, so check staging from the host): + +```bash +# 2015/16 national row exists +curl -sL "https://www.schoolcompare.co.uk/api/national-averages" | python3 -c "import json,sys; d=json.load(sys.stdin); assert any(r['year']==201516 and r['primary'] for r in d['by_year']), '2015/16 missing'; print('201516 ok')" +# 2021/22 school rows exist (Barclay) +curl -sL "https://www.schoolcompare.co.uk/api/schools/138690" | python3 -c "import json,sys; d=json.load(sys.stdin); ys=[r['year'] for r in d['yearly_data']]; assert 202122 in [int(y) for y in ys], ys; print('202122 ok')" +``` + +(The admissions/CI/KS4/rc_* columns aren't API-visible until the backend PR maps them — verify those directly in Postgres from the pipeline host: `select count(*) from marts.fact_admissions where second_preference_offers is not null;` etc.) + +- [ ] **Step 3: Update the spec** — tick off the §5 promotions this PR delivered (edit the spec's promotion list to note "landed in PR #NN") and commit to main via a docs PR or alongside the backend PR. + +--- + +## Out of scope (next plans) + +1. **Backend PR:** map new columns in `backend/models.py`, extend `/api/compare` with supplementary blocks + `national_averages`, computed benchmarks (FSM/EAL/SEN/size medians, disadvantaged national average), CI-based progress banding, report-card label translation (verify against Ofsted toolkit), Ofsted provider-page URLs, graded-vs-ungraded surfacing. +2. **Frontend PR:** rebuild `/compare` per the mockups + e2e journeys (promotion gate). +3. **Separate bug fix:** third school's series not rendering on the current production chart. +4. **Post-v1 (spec):** census ethnicity/young-carer promotion, IDACI display, attendance section, gender-split/absence tier-2 measures. +5. **Already in marts, no work needed:** KS4 EBacc entry/APS, grade 5+ English & maths, Progress 8 CIs — `fact_ks4_performance` carries them today; only the backend needs to expose them. diff --git a/docs/superpowers/plans/2026-07-13-compare-api-enrichment.md b/docs/superpowers/plans/2026-07-13-compare-api-enrichment.md new file mode 100644 index 0000000..4a55d83 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-compare-api-enrichment.md @@ -0,0 +1,399 @@ +# Compare API Enrichment (Backend PR) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Expose the PR #32 data through the API so the redesigned compare screen can be built: enrich `/api/compare` with supplementary blocks + national averages + computed benchmarks, translate Ofsted report-card codes to labels, and surface the new mart columns (spec §6, §8 of `docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md`). + +**Architecture:** All changes are additive API fields — existing consumers keep working. One small dbt change rides along: `fact_performance` (the combined KS2+KS4 mart the backend's `_MAIN_QUERY` reads) enumerates columns explicitly and was not extended in PR #32, so the new KS2 CI and KS4 banding columns must be threaded through it here. Everything else is backend Python: `models.py` mappings, `data_loader` query/supplementary additions, an Ofsted label dictionary (gias_codes pattern), and `/api/compare` composition. + +**Tech Stack:** FastAPI, SQLAlchemy, pandas; dbt (one model); pytest via `python -m pytest backend/tests -q` (CI installs `requirements.txt pytest "httpx<0.28"`; locally use `uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests -q`). + +## Global Constraints + +- **Never push to `main`.** Branch: `feat/compare-api-enrichment`. +- **Additive only** to API responses; never rename/remove existing fields (frontend + e2e depend on them). +- **Report-card scale labels are the live-sampled vocabulary** (evidence in `pipeline/scripts/diagnose_compare_gaps.py`): `1=Exceptional, 2=Strong standard, 3=Expected standard, 4=Needs attention, 5=Urgent improvement`. Never "Attention needed". Safeguarding is boolean met/not-met, never counted as a graded area. +- **Ofsted links** are always the provider page `https://reports.ofsted.gov.uk/provider/21/{urn}` (spec §5) labelled as the school's Ofsted page. +- **Benchmark provenance** (spec §8.6): computed values are "state-school average (computed from our dataset)" — the API must expose them under a `benchmarks` key, clearly separate from official `national_averages`. +- TDD: each behaviour lands with a failing test first, in `backend/tests/` following the `test_school_details.py` pattern (pandas fixture + monkeypatched `load_school_data` + `TestClient`). +- Deploy note for the PR body: the new API fields return NULL/empty until prod's DAGs have run post-promotion. + +--- + +### Task 0: Branch + +- [ ] `git checkout main && git pull && git checkout -b feat/compare-api-enrichment` (commit this plan file on the branch). + +--- + +### Task 1: Thread PR #32 columns through `fact_performance` + +**Files:** +- Modify: `pipeline/transform/models/marts/fact_performance.sql` +- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (fact_performance block, if it has one — add the columns wherever the model's other columns are listed; if the model has no column list there, skip the yml) + +**Interfaces:** +- Produces (for `_MAIN_QUERY` in Task 4): `ks2.*` CI columns and `ks4.progress_8_banding`, `ks4.attainment_8_disadvantage_gap`, `ks4.progress_8_disadvantage_gap` on `marts.fact_performance`. + +- [ ] **Step 1:** In `fact_performance.sql`, after `ks2.reading_progress,` add `ks2.reading_progress_lower_ci,` and `ks2.reading_progress_upper_ci,`; after `ks2.writing_progress,` add `ks2.writing_progress_lower_ci,`, `ks2.writing_progress_upper_ci,`, `ks2.writing_working_towards_pct,`; after `ks2.maths_progress,` add `ks2.maths_progress_lower_ci,`, `ks2.maths_progress_upper_ci,`. In the KS4 section, after the `ks4.progress_8_upper_ci`-equivalent line (locate the Progress 8 block) add: + +```sql + ks4.progress_8_banding, + ks4.attainment_8_disadvantage_gap, + ks4.progress_8_disadvantage_gap, +``` + +- [ ] **Step 2:** Parse gate: `cd pipeline/transform && uv run --with dbt-postgres python -m dbt.cli.main parse --profiles-dir .` → exit 0. + +- [ ] **Step 3:** Commit: `feat(pipeline): thread compare-foundation columns through fact_performance` + +--- + +### Task 2: ORM mappings for the new mart columns + +**Files:** +- Modify: `backend/models.py` (`KS2Performance` after `maths_progress`; `FactAdmissions` after `first_preference_offers`) +- Test: none (declarative mappings; covered by Task 4's query tests) + +**Interfaces:** +- Produces attributes used by Task 4: `KS2Performance.reading_progress_lower_ci` … `maths_progress_upper_ci`, `writing_working_towards_pct` (Float); `FactAdmissions.total_offers`, `.second_preference_offers`, `.third_preference_offers`, `.cross_la_applications`, `.cross_la_offers` (Integer). + +- [ ] **Step 1:** Add to `KS2Performance` (next to the existing progress columns): + +```python + reading_progress_lower_ci = Column(Float) + reading_progress_upper_ci = Column(Float) + writing_progress_lower_ci = Column(Float) + writing_progress_upper_ci = Column(Float) + writing_working_towards_pct = Column(Float) + maths_progress_lower_ci = Column(Float) + maths_progress_upper_ci = Column(Float) +``` + +Add to `FactAdmissions` (after `first_preference_offers`): + +```python + total_offers = Column(Integer) + second_preference_offers = Column(Integer) + third_preference_offers = Column(Integer) + cross_la_applications = Column(Integer) + cross_la_offers = Column(Integer) +``` + +(`FactOfstedInspection` already maps all `rc_*` columns with the right types — verify, don't change.) + +- [ ] **Step 2:** Commit: `feat(api): map compare-foundation mart columns` + +--- + +### Task 3: Ofsted label dictionary + provider URL (TDD) + +**Files:** +- Create: `backend/ofsted_codes.py` +- Test: `backend/tests/test_ofsted_codes.py` + +**Interfaces:** +- Produces for Task 4: `REPORT_CARD_GRADE_NAMES: dict[int, str]`, `report_card_labels(ofsted: dict) -> dict` (returns `{area_key: {"code": int, "label": str}}` for the non-null `rc_*` grade fields, excluding safeguarding), `ofsted_page_url(urn: int) -> str`. + +- [ ] **Step 1: Failing tests** + +```python +"""Report-card code translation uses the live-sampled Ofsted vocabulary +(pipeline/scripts/diagnose_compare_gaps.py TASK 7 VALUE SAMPLE): +Exceptional / Strong standard / Expected standard / Needs attention / +Urgent improvement — never the consultation draft's 'Attention needed'.""" +from backend.ofsted_codes import ( + REPORT_CARD_GRADE_NAMES, report_card_labels, ofsted_page_url, +) + + +def test_scale_is_sampled_vocabulary(): + assert REPORT_CARD_GRADE_NAMES == { + 1: "Exceptional", + 2: "Strong standard", + 3: "Expected standard", + 4: "Needs attention", + 5: "Urgent improvement", + } + + +def test_labels_only_for_populated_areas_and_never_safeguarding(): + ofsted = { + "rc_achievement": 2, + "rc_inclusion": 3, + "rc_attendance_behaviour": 4, + "rc_early_years": None, + "rc_safeguarding_met": True, + "overall_effectiveness": None, + } + labels = report_card_labels(ofsted) + assert labels == { + "rc_achievement": {"code": 2, "label": "Strong standard"}, + "rc_inclusion": {"code": 3, "label": "Expected standard"}, + "rc_attendance_behaviour": {"code": 4, "label": "Needs attention"}, + } + + +def test_unknown_code_is_skipped_not_crashed(): + assert report_card_labels({"rc_achievement": 9}) == {} + + +def test_provider_url(): + assert ofsted_page_url(138690) == "https://reports.ofsted.gov.uk/provider/21/138690" +``` + +- [ ] **Step 2:** Run `uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests/test_ofsted_codes.py -q` → FAIL (module missing). + +- [ ] **Step 3: Implement `backend/ofsted_codes.py`** + +```python +"""Ofsted renewed-framework (Nov 2025) report-card code translation. + +Scale labels are the live-sampled vocabulary from the Ofsted MI file +(see pipeline/scripts/diagnose_compare_gaps.py, TASK 7 VALUE SAMPLE) — +verified against real data, not the consultation draft. +""" + +REPORT_CARD_GRADE_NAMES = { + 1: "Exceptional", + 2: "Strong standard", + 3: "Expected standard", + 4: "Needs attention", + 5: "Urgent improvement", +} + +# Graded evaluation areas only — safeguarding is a separate boolean +# judgement and must never appear in grade counts or label maps. +_RC_AREA_KEYS = ( + "rc_inclusion", + "rc_curriculum_teaching", + "rc_achievement", + "rc_attendance_behaviour", + "rc_personal_development", + "rc_leadership_governance", + "rc_early_years", + "rc_sixth_form", +) + + +def report_card_labels(ofsted: dict) -> dict: + """{area_key: {code, label}} for populated, known-valued rc_* areas.""" + out = {} + for key in _RC_AREA_KEYS: + code = ofsted.get(key) + label = REPORT_CARD_GRADE_NAMES.get(code) + if code is not None and label is not None: + out[key] = {"code": code, "label": label} + return out + + +def ofsted_page_url(urn: int) -> str: + """The school's page on ofsted.gov.uk (all its reports live there — + we never deep-link an individual report; spec §5).""" + return f"https://reports.ofsted.gov.uk/provider/21/{urn}" +``` + +- [ ] **Step 4:** Re-run the test file → 4 passed. Run the full suite (same command, `backend/tests -q`) → all pass. + +- [ ] **Step 5:** Commit: `feat(api): Ofsted report-card labels and provider-page URL` + +--- + +### Task 4: data_loader — query columns + richer supplementary blocks (TDD) + +**Files:** +- Modify: `backend/data_loader.py` (`_MAIN_QUERY` ~line 153; `get_supplementary_data` ~line 460) +- Test: `backend/tests/test_supplementary_enrichment.py` + +**Interfaces:** +- `_MAIN_QUERY` additionally selects (KS2 block, after `p.maths_progress`): `p.reading_progress_lower_ci, p.reading_progress_upper_ci, p.writing_progress_lower_ci, p.writing_progress_upper_ci, p.writing_working_towards_pct, p.maths_progress_lower_ci, p.maths_progress_upper_ci`; (KS4 block, after the Progress 8 CI columns): `p.progress_8_banding, p.attainment_8_disadvantage_gap, p.progress_8_disadvantage_gap`. Note `_MAIN_QUERY_NO_SIXTH_FORM`/`_MAIN_QUERY_LEGACY_NAMES` are string-derived from `_MAIN_QUERY` (lines 259-270) and inherit automatically — verify the assertions there still hold. +- `get_supplementary_data(db, urn)["admissions"]` rows additionally carry: `total_offers`, `second_preference_offers`, `third_preference_offers`, `cross_la_applications`, `cross_la_offers` (add to `_admissions_row`). +- `get_supplementary_data(db, urn)["ofsted"]` additionally carries: `report_card` (the `report_card_labels(...)` dict, `{}` when no rc data), `ofsted_page_url`, and `grade_source`: `"graded"` when `overall_effectiveness` came from the graded column, `"ungraded_carried_forward"` when the fallback `ungraded_grade` supplied it, `None` when neither. + +- [ ] **Step 1: Failing tests** — construct a fake Ofsted row object (simple `types.SimpleNamespace` with the model's attributes) and call the block-building logic via `get_supplementary_data` with a stubbed session (follow how existing tests stub the db; if none do, factor the ofsted-dict construction into a pure helper `_ofsted_block(o, urn)` and test that directly — preferred): + +```python +import types +from backend.data_loader import _ofsted_block + + +def _row(**kw): + base = dict( + framework="RC", inspection_date=None, inspection_type=None, + overall_effectiveness=None, quality_of_education=None, + behaviour_attitudes=None, personal_development=None, + leadership_management=None, early_years_provision=None, + sixth_form_provision=None, ungraded_outcome=None, ungraded_grade=None, + rc_safeguarding_met=None, rc_inclusion=None, rc_curriculum_teaching=None, + rc_achievement=None, rc_attendance_behaviour=None, + rc_personal_development=None, rc_leadership_governance=None, + rc_early_years=None, rc_sixth_form=None, report_url=None, + ) + base.update(kw) + return types.SimpleNamespace(**base) + + +def test_report_card_block_and_provider_url(): + o = _row(rc_achievement=2, rc_inclusion=3, rc_safeguarding_met=True) + block = _ofsted_block(o, urn=100140) + assert block["report_card"]["rc_achievement"]["label"] == "Strong standard" + assert "rc_safeguarding_met" not in block["report_card"] + assert block["rc_safeguarding_met"] is True + assert block["ofsted_page_url"] == "https://reports.ofsted.gov.uk/provider/21/100140" + + +def test_grade_source_graded_vs_carried_forward(): + assert _ofsted_block(_row(overall_effectiveness=1), urn=1)["grade_source"] == "graded" + carried = _ofsted_block(_row(ungraded_grade=2), urn=1) + assert carried["grade_source"] == "ungraded_carried_forward" + assert carried["overall_effectiveness"] == 2 + assert _ofsted_block(_row(), urn=1)["grade_source"] is None + + +def test_admissions_row_new_fields(): + from backend.data_loader import _admissions_row_dict + a = types.SimpleNamespace( + year=202627, school_phase="Primary", places_offered=80, + total_applications=185, first_preference_applications=74, + first_preference_offers=74, first_preference_offer_pct=100.0, + oversubscription_ratio=0.925, oversubscribed=False, + total_offers=80, second_preference_offers=4, third_preference_offers=2, + cross_la_applications=12, cross_la_offers=3, + ) + d = _admissions_row_dict(a) + for k in ("total_offers", "second_preference_offers", "third_preference_offers", + "cross_la_applications", "cross_la_offers"): + assert d[k] == getattr(a, k) +``` + +- [ ] **Step 2:** Run → FAIL (helpers don't exist). + +- [ ] **Step 3: Implement.** Refactor the existing inline ofsted-dict construction in `get_supplementary_data` into a module-level `_ofsted_block(o, urn)` that produces the existing keys **unchanged** plus the three new ones (`report_card` via `report_card_labels(...)` from Task 3, `ofsted_page_url` via `ofsted_page_url(urn)`, `grade_source` per the interface rule — derived from which source supplied `overall_effectiveness`). Rename/extract the local `_admissions_row` into module-level `_admissions_row_dict(a)` and append the five new fields. Add the ten new columns to `_MAIN_QUERY` exactly as the interface lists them. `get_supplementary_data` calls both helpers; its external shape gains only additive keys. + +- [ ] **Step 4:** Full suite → all pass (existing `test_school_details.py` etc. must not break; if a fixture enumerates yearly-data columns, extend it with the new NaN columns as needed). + +- [ ] **Step 5:** Commit: `feat(api): expose progress CIs, KS4 banding/gaps, admissions detail, report-card labels` + +--- + +### Task 5: Computed benchmarks helper (TDD) + +**Files:** +- Modify: `backend/data_loader.py` (new function) +- Test: `backend/tests/test_benchmarks.py` + +**Interfaces:** +- Produces for Task 6: `compute_benchmarks(df) -> dict` — pure function over the main dataframe (latest year, state schools), shape: + +```python +{ + "source": "state-school average (computed from our dataset)", + "year": 202425, + "primary": { + "disadvantaged_rwm_expected_pct": 46.1, # weighted by eligible_pupils + "eal_pct": 22.3, # median + "sen_support_pct": 14.0, # median + "disadvantaged_pct": 24.8, # median (FSM6 proxy) + "median_pupils": 281, # median school size + }, + "secondary": { "median_pupils": 1024, "eal_pct": ..., "sen_support_pct": ..., "disadvantaged_pct": ... }, +} +``` + +- [ ] **Step 1: Failing tests** — build a small synthetic df (6 primary rows with known eligible_pupils/rwm_expected_disadvantaged_pct so the weighted average is hand-checkable; a couple of secondary rows flagged by non-null `attainment_8_score`), assert: weighted disadvantaged average matches hand computation (not the unweighted mean), medians ignore NaN, secondary block lacks the disadvantaged-RWM key, latest-year filtering (rows from an older year must not affect results), and empty df → `{}`. + +- [ ] **Step 2:** Run → FAIL. + +- [ ] **Step 3: Implement** in `data_loader.py`: + +```python +def compute_benchmarks(df: pd.DataFrame) -> dict: + """State-school benchmarks computed from our dataset (spec §5/§8.6). + These are NOT official DfE figures — consumers must label them + 'state-school average (computed from our dataset)'.""" + if df.empty or "year" not in df.columns: + return {} + latest_year = df["year"].max() + d = df[df["year"] == latest_year] + if d.empty: + return {} + is_secondary = d["attainment_8_score"].notna() if "attainment_8_score" in d.columns else pd.Series(False, index=d.index) + prim, sec = d[~is_secondary], d[is_secondary] + + def _median(sub, col): + if col not in sub.columns: + return None + v = sub[col].median() + return round(float(v), 1) if pd.notna(v) else None + + def _weighted_disadvantaged(sub): + if not {"rwm_expected_disadvantaged_pct", "eligible_pupils"} <= set(sub.columns): + return None + s = sub.dropna(subset=["rwm_expected_disadvantaged_pct", "eligible_pupils"]) + if s.empty or s["eligible_pupils"].sum() == 0: + return None + w = (s["rwm_expected_disadvantaged_pct"] * s["eligible_pupils"]).sum() / s["eligible_pupils"].sum() + return round(float(w), 1) + + def _block(sub, with_disadvantaged): + block = { + "eal_pct": _median(sub, "eal_pct"), + "sen_support_pct": _median(sub, "sen_support_pct"), + "disadvantaged_pct": _median(sub, "disadvantaged_pct"), + "median_pupils": int(sub["total_pupils"].median()) if "total_pupils" in sub.columns and pd.notna(sub["total_pupils"].median()) else None, + } + if with_disadvantaged: + block["disadvantaged_rwm_expected_pct"] = _weighted_disadvantaged(sub) + return block + + return { + "source": "state-school average (computed from our dataset)", + "year": int(latest_year), + "primary": _block(prim, with_disadvantaged=True), + "secondary": _block(sec, with_disadvantaged=False), + } +``` + +(Adapt column presence to the real df — `sen_support_pct` reaches the df via `_MAIN_QUERY`; confirm and add it there if the KS2 block doesn't already select it, mirroring Task 4's additions.) + +- [ ] **Step 4:** Full suite → pass. **Step 5:** Commit: `feat(api): computed state-school benchmarks` + +--- + +### Task 6: Enrich `/api/compare` + expose GPS/science national averages (TDD) + +**Files:** +- Modify: `backend/app.py` (`compare_schools` ~line 636; `get_national_averages` ~line 730) +- Test: `backend/tests/test_compare_enrichment.py` + +**Interfaces (response additions, all additive):** +- `/api/compare` top level gains: `"national_averages"` (same payload the `/api/national-averages` endpoint returns — extract the endpoint body into a helper `_national_averages_payload(df)` and reuse; do not duplicate the logic) and `"benchmarks"` (Task 5's `compute_benchmarks(df)`). +- Each `comparison[urn]` gains: `"ofsted"`, `"census"`, `"admissions"`, `"admissions_history"`, `"deprivation"` from `get_supplementary_data` (one `SessionLocal()` for the whole request, closed in `finally`; on exception the five keys are `None`/`[]` — mirror the detail endpoint's defensive pattern at app.py:583-590). +- `get_national_averages`' KS2 metric list gains `"gps_expected_pct", "gps_high_pct", "science_expected_pct"` so the England ticks for GPS/science flow once the data exists. + +- [ ] **Step 1: Failing tests** — monkeypatch `load_school_data` with a two-school primary df (reuse/extend the fixture style of `test_school_details.py`) and monkeypatch `get_supplementary_data` to a canned dict; assert on `TestClient(app).get("/api/compare?urns=...")`: + - response keeps the existing shape (`comparison[urn]["school_info"]["rwm_expected_pct"]` etc.), + - each school gains the five supplementary keys (canned values round-tripped), + - top-level `national_averages` and `benchmarks` present; `benchmarks["source"]` is the exact provenance string, + - a supplementary-layer exception (monkeypatched to raise) degrades to `ofsted: None` etc. with HTTP 200, + - `/api/national-averages` includes `gps_expected_pct` in the primary block when the df/national table provides it (monkeypatch the national-averages source the endpoint reads). + +- [ ] **Step 2:** Run → FAIL. **Step 3:** Implement per the interfaces. **Step 4:** Full suite → pass. + +- [ ] **Step 5:** Commit: `feat(api): compare endpoint carries supplementary blocks, national averages and benchmarks` + +--- + +### Task 7: PR + verification + +- [ ] **Step 1:** Full suite one more time + `uv run --with pyyaml python3 -c "import yaml; yaml.safe_load(open('.gitea/workflows/deploy.yml'))"` sanity is NOT needed (no workflow changes) — instead run the dbt parse gate again (Task 1 file). +- [ ] **Step 2:** Push, open PR via the Gitea API (credential-helper basic auth). PR body: the new response shapes (one JSON sketch), the reused-not-duplicated national-averages helper, the provenance rule for benchmarks, deploy note (fields NULL until prod DAGs run post-promotion), and that no e2e change is needed (no user-facing behaviour changes — the compare UI still reads the old fields; the frontend PR carries the journey updates). +- [ ] **Step 3:** After merge + staging deploy: `curl -s https://stx.schoolcompare.co.uk/api/compare?urns=138690,100140 | python3 -m json.tool | head -80` — verify the new keys and that `benchmarks.primary.disadvantaged_rwm_expected_pct` is plausible (~45-47). Verify `/api/national-averages` now carries `gps_expected_pct`/`science_expected_pct` (values or honest nulls if DfE suppresses them at national level). + +--- + +## Out of scope + +- Frontend rebuild + e2e journeys (next PR — consumes everything this PR exposes). +- `schemas.py` METRIC_DEFINITIONS additions for the trends picker (frontend PR decides which of the new columns become picker metrics). +- CI-based progress banding logic (frontend computes Above/Average/Below from the CI columns; historical years only). diff --git a/docs/superpowers/plans/2026-07-13-compare-frontend-rebuild.md b/docs/superpowers/plans/2026-07-13-compare-frontend-rebuild.md new file mode 100644 index 0000000..dcb165d --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-compare-frontend-rebuild.md @@ -0,0 +1,287 @@ +# Compare Screen Frontend Rebuild Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rebuild `/compare` in the Next.js app to match the approved mockups — parent-first sections (At a glance / Ofsted / Academics / Getting a place / Who goes there / Explore trends), England-average anchoring with provenance-correct labels, mobile-first measure-first layout — consuming the enriched `/api/compare` payload from PR #34, with e2e journeys updated in the same PR (they are the promotion gate). + +**Architecture:** `ComparisonView` becomes an assembly of section components fed by one enriched fetch. All comprehension rules from the two expert reviews live in a pure, jest-tested module (`lib/compareLogic.ts`) — components stay presentational. The mockups are committed at `docs/superpowers/specs/mockups/compare-desktop.html` and `compare-mobile.html`: **all user-facing copy (labels, tooltips, chips, footnote wording) is taken verbatim from them** — they carry two rounds of education-expert review; do not paraphrase. + +**Tech Stack:** Next.js (app router, SSR page + client view), CSS modules, Chart.js (existing `ComparisonChart`), Jest (`npm test` in `nextjs-app/`), Playwright e2e (`e2e/`). + +## Global Constraints + +- **Never push to `main`.** Branch: `feat/compare-frontend-rebuild`. +- **Copy is expert-reviewed:** take it verbatim from the committed mockups. Binding rules (spec §8): Ofsted scale labels come from the API's `report_card[..].label` (never hardcode area labels beyond the mockups'); official DfE numbers say "England average", computed ones say "state-school average (computed from our dataset)"; the 2021/22 chart gap note says "DfE didn't publish school-level figures for 2021/22"; never derive an overall grade from report-card areas; safeguarding never counts as a graded area; "Latest Ofsted inspection", "EHC plans", "at or above capacity", "Over 1 in 4", "first choice (officially 'first preference')". +- **Mobile-first:** the measure-first stacked layout (mobile mockup) is the base CSS; the desktop label-column grid is the `min-width` enhancement. +- **URL contract unchanged:** `?urns=` (and `metric=` now scoped to Explore trends) keep working; share flow, `useComparison` basket, phase tabs, and `compare_viewed`/`compare_metric_changed` analytics events are preserved. +- **Do not run a local server** (CLAUDE.md); verification = jest + `tsc` + the e2e suite against staging after merge. e2e must pass on **staging data** — remember staging has partial history: assert against the *latest* year, never oldest. +- Existing `/api/compare` consumers elsewhere in the app (SchoolDetail links, toasts) must not break — the response is additive, and this PR only rewrites the compare page's own components. +- **Post-v1 (do not build):** IDACI, attendance section, gender-split/absence tier-2 measures, finance (spec §4). + +--- + +### Task 0: Branch + design sources + +- [ ] `git checkout main && git pull && git checkout -b feat/compare-frontend-rebuild` +- [ ] The mockups and this plan are already in the working tree (`docs/superpowers/specs/mockups/compare-{desktop,mobile}.html`) — commit them: `docs: compare mockups as frontend design source + rebuild plan` + +--- + +### Task 1: Types for the enriched payload + +**Files:** +- Modify: `nextjs-app/lib/types.ts` (extend `SchoolResult`, `ComparisonData`, `ComparisonResponse` — located around lines 293-314) + +**Interfaces (produced for every later task):** + +```ts +export interface ReportCardEntry { code: number; label: string; } + +export interface OfstedBlock { + framework: string | null; + inspection_date: string | null; + inspection_type: string | null; + overall_effectiveness: number | null; + grade_source: 'graded' | 'ungraded_carried_forward' | null; + quality_of_education: number | null; + behaviour_attitudes: number | null; + personal_development: number | null; + leadership_management: number | null; + early_years_provision: number | null; + sixth_form_provision: number | null; + rc_safeguarding_met: boolean | null; + report_card: Record; + ofsted_page_url: string; + report_url: string | null; +} + +export interface CensusBlock { + year: number | null; total_pupils: number | null; + female_pupils: number | null; male_pupils: number | null; + fsm_pct: number | null; eal_pct: number | null; +} + +export interface AdmissionsRow { + year: number; school_phase: string | null; + places_offered: number | null; total_applications: number | null; + first_preference_applications: number | null; first_preference_offers: number | null; + first_preference_offer_pct: number | null; oversubscription_ratio: number | null; + oversubscribed: boolean | null; + total_offers: number | null; second_preference_offers: number | null; + third_preference_offers: number | null; + cross_la_applications: number | null; cross_la_offers: number | null; +} + +export interface DeprivationBlock { + lsoa_code: string | null; idaci_score: number | null; idaci_decile: number | null; +} + +export interface BenchmarkBlock { + eal_pct: number | null; sen_support_pct: number | null; + disadvantaged_pct: number | null; median_pupils: number | null; + disadvantaged_rwm_expected_pct?: number | null; +} + +export interface Benchmarks { + source: string; year: number; + primary: BenchmarkBlock; secondary: BenchmarkBlock; +} + +export interface NationalAverages { + year: number; + primary: Record; + secondary: Record; + by_year: Array<{ year: number; primary: Record; secondary: Record }>; +} +``` + +- [ ] **Step 1:** Add the interfaces above; extend `ComparisonData` with optional `ofsted?: OfstedBlock | null; census?: CensusBlock | null; admissions?: AdmissionsRow | null; admissions_history?: AdmissionsRow[]; deprivation?: DeprivationBlock | null;` and `ComparisonResponse` with `national_averages?: NationalAverages; benchmarks?: Benchmarks;` (optional so the UI degrades on an old backend). Extend `SchoolResult` with the ten new yearly columns (`reading_progress_lower_ci` … `maths_progress_upper_ci`, `writing_working_towards_pct`, `progress_8_banding: string | null`, `attainment_8_disadvantage_gap`, `progress_8_disadvantage_gap`). +- [ ] **Step 2:** `cd nextjs-app && npx tsc --noEmit` → clean. Commit: `feat(compare): types for enriched comparison payload` + +--- + +### Task 2: `lib/compareLogic.ts` — the comprehension rules, jest-tested + +**Files:** +- Create: `nextjs-app/lib/compareLogic.ts` +- Test: `nextjs-app/__tests__/lib/compareLogic.test.ts` + +**Interfaces (produced):** + +```ts +export type Verdict = 'above' | 'close' | 'below'; +export function verdict(value: number, anchor: number, tolerance?: number): Verdict; // default tolerance 2pp + +// Report-card summary per spec §4.2: count graded areas per label (best +// first), NAME any 'Needs attention'/'Urgent improvement' area, safeguarding +// separate, "No areas need attention" reassurance when applicable. +export interface ReportCardSummary { + counts: Array<{ label: string; count: number }>; // best grade first + problems: Array<{ areaLabel: string; label: string }>; // named, never counted-away + safeguarding: 'met' | 'not_met' | null; + allClear: boolean; +} +export function summariseReportCard(ofsted: OfstedBlock): ReportCardSummary; + +// One display model for all three inspection regimes. +export type OfstedDisplay = + | { kind: 'none' } + | { kind: 'graded'; grade: number; gradeLabel: string; carriedForward: false } + | { kind: 'carried_forward'; grade: number; gradeLabel: string; carriedForward: true } + | { kind: 'report_card'; summary: ReportCardSummary }; +export function ofstedDisplay(ofsted: OfstedBlock | null | undefined): OfstedDisplay; +export const OFSTED_LEGACY_GRADES: Record; // 1 Outstanding, 2 Good, 3 Requires improvement, 4 Inadequate + +// Human-readable area label from an rc_ key: 'rc_attendance_behaviour' → +// 'Attendance & behaviour' (mapping table copied from the mockups' area rows). +export function rcAreaLabel(key: string): string; + +// Admissions, one consistent chip metric (first-preference success). +export interface AdmissionsSummary { + firstPrefPct: number | null; + chip: { tone: 'good' | 'warn' | 'neutral'; text: string } | null; // "97% of first choices offered" / "Over 1 in 4 first choices missed out" wording per mockups + interest: string | null; // "Named on 457 forms · 180 places" +} +export function summariseAdmissions(a: AdmissionsRow | null | undefined): AdmissionsSummary; + +// CI-based progress band for historical years (null when no CI published). +export function progressBand(score: number | null, lower: number | null, upper: number | null): + 'above' | 'average' | 'below' | null; // CI entirely >0 → above; entirely <0 → below; straddles → average + +// Dot-strip geometry (used by the DotStrip component; pure for testing). +export interface StripPoint { pos: number; labelAbove: boolean; value: number; schoolIndex: number; } +export function stripPositions(values: Array, min: number, max: number): StripPoint[]; +// pos = (v-min)/(max-min)*100 clamped 0..100; labels within 4% of range of a +// lower neighbour flip above (the mockups' collision nudge). +``` + +- [ ] **Step 1: Failing tests** covering, at minimum: + - `summariseReportCard`: 4 Strong + 2 Expected + 1 Needs-attention + safeguarding met → counts `[Strong standard×4, Expected standard×2]`, `problems=[{areaLabel:'Attendance & behaviour', label:'Needs attention'}]`, `allClear=false`; safeguarding NEVER in counts; all-Expected+met → `allClear=true`; labels come from the input's `.label` (assert the function never invents "Attention needed"). + - `ofstedDisplay`: report_card present → `kind:'report_card'` even if a legacy grade also exists; `grade_source:'ungraded_carried_forward'` → `carriedForward:true`; null → `'none'`. + - `summariseAdmissions`: 73% → warn chip text `Over 1 in 4 first choices missed out`; 97% → good chip `97% of first choices offered`; 100% → `All first choices offered`; interest string `Named on 342 forms · 120 places`; nulls → null chip. + - `progressBand`: (1.2, 0.4, 2.0)→above; (-1.2, -2.0, -0.4)→below; (0.3, -0.5, 1.1)→average; missing CI → null. + - `stripPositions`: 100–120 domain maps 106→30; values 91 and 92 on 0–100 → second label flips above; nulls skipped. + - `verdict`: 87 vs 62 → above; 61 vs 62 → close (within 2pp); 40 vs 62 → below. +- [ ] **Step 2:** `cd nextjs-app && npm test -- compareLogic` → FAIL. **Step 3:** implement. **Step 4:** pass + `tsc` clean. **Step 5:** Commit: `feat(compare): comprehension logic (report cards, admissions, verdicts, strips)` + +--- + +### Task 3: `DotStrip` component + +**Files:** +- Create: `nextjs-app/components/DotStrip.tsx`, `nextjs-app/components/DotStrip.module.css` + +**Interfaces:** + +```ts +export interface DotStripProps { + label: string; + values: Array; // one per school, school order = chart colour order + anchor?: { value: number; label: string } | null; // e.g. {62, "England 62%"} — omit when benchmark absent + min?: number; max?: number; // default 0..100 + unit?: string; // default '%' + tip?: string; // title tooltip on the label + note?: string; // e.g. "(teacher-assessed)" suffix handled by caller in label +} +``` + +- [ ] Render per the mockups' `.strip-row` anatomy: label row, 4px track, England tick + tick label, 16px dots coloured by `CHART_COLORS[index]` with white ring, value labels below (flipped above on collision via `stripPositions`). `role="img"` + `aria-label` enumerating anchor and each school's value (copy the aria pattern from the mockups). CSS module mirrors the mockup styles using the app's CSS variables (`--border-light`, `--text-muted`, etc.). +- [ ] Jest: render with `@testing-library/react` (already configured — see `__tests__/components/SecondarySchoolRow.test.tsx` for the harness pattern): asserts aria-label content, tick present when anchor given, absent otherwise. +- [ ] Commit: `feat(compare): DotStrip with England-average anchor` + +--- + +### Task 4: Section components — At a glance, Ofsted, Getting a place, Who goes there + +**Files:** +- Create: `nextjs-app/components/compare/CompareAtAGlance.tsx` (+ `.module.css`) +- Create: `nextjs-app/components/compare/CompareOfsted.tsx` +- Create: `nextjs-app/components/compare/CompareAdmissions.tsx` +- Create: `nextjs-app/components/compare/CompareCommunity.tsx` +- Create: `nextjs-app/components/compare/compareSections.module.css` (shared measure-first grid) +- Test: `nextjs-app/__tests__/components/CompareOfsted.test.tsx` + +**Shared layout contract (all four):** props `{ schools: School[]; data: Record; benchmarks?: Benchmarks; nationalAverages?: NationalAverages }`. Base CSS is the mobile mockup's measure-first stack (`.measure` card → `.srow` per school with colour dot + short name + value + chip + note); at `min-width: 761px` it becomes the desktop mockup's grid (200px row-label column + one column per school). Section headers use the existing `.section-title` idiom; every section carries its mockup "how" line verbatim. + +**Content per section = the mockups, row for row.** Structure/tone rules already encoded in Task 2's helpers: +- *At a glance*: Latest Ofsted inspection row (badge via `ofstedDisplay`; report-card case renders `ReportCardSummary` chips — counts best-first + named problem chips + safeguarding line); expected-standard row (big % + `verdict` chip vs `national_averages.primary.rwm_expected_pct`, small "England average N%"); Getting a place row (chip from `summariseAdmissions`, note = `interest`); Size row (pupils + "at or above capacity"/"N% full" from census/capacity, vs `benchmarks.*.median_pupils` for "larger/smaller than average" phrasing). +- *Ofsted*: the section's `how` paragraph (regime explanation + non-comparability + "Expected standard" disambiguation) verbatim from the desktop mockup; Result row; Inspected row (date + "4+ years ago" chip when >4y, computed from `inspection_date`); Judgement detail row — **one chip-list grammar for both regimes** (legacy subgrades via `OFSTED_LEGACY_GRADES`; report card via `report_card` labels; "We don't hold area-by-area detail for this inspection" when neither); Ofsted page row linking `ofsted_page_url` ("'s Ofsted page →"). +- *Getting a place*: `how` paragraph (first preference/equal preference/offer-day caveats) verbatim; Interest row; first-choice success row with mini bar; "What this means" row (distance note: "check the school's admission criteria (for most non-faith primaries, distance decides)" only when oversubscribed). +- *Who goes there*: pupils-on-roll (census + capacity), girls/boys, FSM (chip vs `benchmarks` with "state-school average" wording), EAL, SEN (tooltip incl. "EHC plans" + specialist-provision note), faith, ages · nursery, run by (trust name or " council"). + +- [ ] **Step 1:** Failing jest test for `CompareOfsted` (the riskiest): given one graded school, one carried-forward, one report-card school → asserts the three Result cells ("Outstanding" badge; badge + carried-forward marker; "Report card" + no invented overall grade), the chip-list judgement rows, and the comparability note appearing only for the mixed case. +- [ ] **Step 2-4:** Implement all four sections; test passes; `tsc` clean; `npm test` full suite green. +- [ ] **Step 5:** Commit: `feat(compare): at-a-glance, Ofsted, admissions and community sections` + +--- + +### Task 5: `CompareAcademics` — strips + More measures + +**Files:** +- Create: `nextjs-app/components/compare/CompareAcademics.tsx` +- Test: extend `nextjs-app/__tests__/lib/compareLogic.test.ts` with the metric-extraction helper below + +**Interfaces:** +- Add to `compareLogic.ts`: `latestValues(data, urns, metricKey) => Array` (latest non-null yearly value per school) — tested. + +- [ ] Tier 1 strips (always visible), each a `DotStrip` with the England anchor from `national_averages.primary`: RWM expected, Reading, Writing, Maths, "Working at a higher standard than expected" (tooltip: composition sentence from the mockups). Section `how` line: "tests and teacher assessments … writing is assessed by teachers, not tested" verbatim. +- [ ] Tier 2 `
` "More measures — grammar, punctuation & spelling, science, average scaled scores": GPS + Science (teacher-assessed, tooltip verbatim) with anchors from `national_averages` **when present, no tick + honest note when null**; scaled scores (reading/maths/GPS) on `min=100 max=120` with the mockups' window caption. +- [ ] Equity row: disadvantaged pupils' RWM per school + chip vs `benchmarks.primary.disadvantaged_rwm_expected_pct` with the "state-school average" wording and small-cohort tooltip verbatim. +- [ ] Secondary phase variant (when active phase is secondary): tier-1 rows are Attainment 8 (anchor `national_averages.secondary.attainment_8_score`), Progress 8 banding (chip showing `progress_8_banding` verbatim — DfE's own label), grade 5+ English & maths %; tier-2: EBacc entry/APS. Measure-first rows (no strips needed for banding). +- [ ] `npm test` + `tsc`; commit: `feat(compare): academics strips with England anchors and More measures` + +--- + +### Task 6: Trends explorer — England line, gap-honest axis, series bug + +**Files:** +- Modify: `nextjs-app/components/ComparisonChart.tsx` +- Create: `nextjs-app/components/compare/TrendsExplorer.tsx` +- Test: `nextjs-app/__tests__/components/ComparisonChart.test.tsx` + +- [ ] **Step 1 (bug first): root-cause the missing third series** seen on production (3 schools in table, 2 lines on chart). Write a failing jest test: 3 schools whose `yearly_data` year values are floats (`202425.0`) vs the labels array — the suspect is the year-matching in `ComparisonChart.tsx:69` (`years.map(...)` built from school 1 only + strict equality against other schools' years). Fix so every school's series renders and years are the union of all schools' years, sorted. +- [ ] **Step 2:** Add optional `nationalByYear?: Record` prop → dashed grey "England average" dataset (colour `--text-muted`, `borderDash:[5,4]`, no fill, `spanGaps:false`). +- [ ] **Step 3:** Gap honesty: x-axis category labels include 2019/20 and 2020/21 as empty slots (band label "tests cancelled 2019/20–2020/21" via a Chart.js annotation-free approach: two category ticks with all-null data and a subtitle note under the chart, copy verbatim: the chart footnote "DfE didn't publish school-level figures for 2021/22" appears when the metric is a KS2 measure and 2021/22 school values are null while the England value exists). `spanGaps:false` on school datasets so dataset gaps break lines. +- [ ] **Step 4:** `TrendsExplorer` wraps the grouped metric picker (existing optgroup structure and `metrics` from `/api/metrics`, existing analytics event) + the chart + the existing year-by-year table, inside a collapsed-by-default `
` ("Explore trends"). Progress metrics annotate cells with `progressBand` chips for years where CIs exist. +- [ ] Tests pass; commit: `feat(compare): trends explorer with England line; fix missing series` + +--- + +### Task 7: Assemble the new `ComparisonView` + +**Files:** +- Rewrite: `nextjs-app/components/ComparisonView.tsx` (+ its `.module.css`) +- Modify: `nextjs-app/app/compare/page.tsx` metadata description (mention Ofsted/admissions, not just KS2) + +- [ ] Preserve intact: `useComparison` basket seeding/URL sync (lines 76-122 of the current file), share handler, phase tabs + auto-detection, `compare_viewed` analytics, empty states, `SchoolSearchModal`, max-4-visible column scroll. Replace the metric-picker/chart/table body with the section stack: sticky school chip bar (mockup `.school-bar`) → `CompareAtAGlance` → `CompareOfsted` → `CompareAcademics` → `CompareAdmissions` → `CompareCommunity` → `TrendsExplorer`. The page-level `metric` URL param now initialises `TrendsExplorer`'s picker only. +- [ ] Top-of-page subtitle + sources footnote verbatim from the mockups (minus the "Mockup" banner), including the suppression rule sentence and provenance sentence. +- [ ] `npm test` full suite + `tsc` clean. Commit: `feat(compare): parent-first compare screen assembly` + +--- + +### Task 8: e2e journeys (the promotion gate) + +**Files:** +- Modify: `e2e/tests/journeys.spec.ts` (the two compare tests, lines ~141-215; extend, don't delete coverage) + +- [ ] Update 'comparing two schools shows both side by side': after loading `/compare?urns=…` assert the new section headings (`At a glance`, `Ofsted inspection`, `How children do academically`, `Getting a place`, `Who goes there`, `Explore trends`), both school names in the sticky bar, at least one England-average tick label (`text=/England \d+%/`), and one provenance string `state-school average` somewhere (benchmarks row). Data-invariant style — no exact numbers (staging data shifts; use latest-year values only). +- [ ] Update the mobile test: 390px viewport, assert measure-first stacking (a `.measure`-card contains all selected school names within one card) and that the trends chart container scrolls (`overflow-x`). +- [ ] Add a report-card presence-agnostic assertion: the Ofsted section renders either a grade badge or "Report card" without an overall grade — i.e. never both an overall-grade badge AND report-card chips for the same school. +- [ ] Run against staging from the host if reachable (`cd e2e && BASE_URL=https://stx.schoolcompare.co.uk npx playwright test -g "compar"`) — staging still runs the OLD UI until this PR merges, so expect failures locally; the authoritative run is the Stage pipeline post-merge. Still commit only after jest+tsc are green. +- [ ] Commit: `test(e2e): compare journeys for the parent-first redesign` + +--- + +### Task 9: PR + post-merge verification + +- [ ] Full gates: `cd nextjs-app && npm test && npx tsc --noEmit`. +- [ ] Push; open PR via Gitea API (credential-helper basic auth). PR body: before/after summary, link to mockups + spec §4/§8, the copy-verbatim rule, the fixed third-series bug, deploy note (needs PR #34's API on the same environment — merge order: #34 first), and that the e2e suite is the staging gate. +- [ ] Post-merge: watch the Stage pipeline — its e2e run against staging is the real verification. Then the human tests staging and promotes (two-stage model). Update memory: compare redesign shipped to staging. + +--- + +## Out of scope + +- IDACI / attendance / gender-absence / finance (post-v1, spec §4). +- Backend changes of any kind (PR #34 must merge first). +- Chart palette overhaul beyond the England-line addition (`CHART_COLORS` swap to the validated trio is a candidate follow-up, flagged not included — it affects every chart in the app). diff --git a/docs/superpowers/plans/2026-07-13-staged-prod-promotion.md b/docs/superpowers/plans/2026-07-13-staged-prod-promotion.md new file mode 100644 index 0000000..4179097 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-staged-prod-promotion.md @@ -0,0 +1,275 @@ +# Staged Production Promotion (Manual Gate) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Merging a PR deploys to staging only; production deployment requires a second, explicit human approval after manual testing on staging. + +**Architecture:** Split the existing single `deploy.yml` pipeline in two. The push-to-main workflow keeps build → staging deploy → e2e gate and **stops there**. A new `promote.yml` runs only on `workflow_dispatch` (the "Run workflow" button in Gitea's Actions UI, supported on this server — Gitea 1.26.4): it verifies the chosen commit passed the staging e2e gate, retags its `:sha-*` images to `:prod` (keeping `:prod-previous` for rollback), and triggers the Portainer prod webhook. Promotion granularity is a main-branch commit: staging always runs the latest main, so you approve a *state of main*, not an individual PR. + +**Tech Stack:** Gitea Actions (1.26.4), Docker buildx imagetools, Portainer webhooks, Gitea commit-status API. + +## Global Constraints + +- **Never push to `main` directly** — this change itself goes through a PR (`chore/staged-prod-promotion` branch). +- Existing image tagging scheme is unchanged: `type=sha` (e.g. `sha-6f925ab`) + `:staging`; promotion still retags `:sha-*` → `:prod` with `:prod-previous` kept as the rollback pointer. +- The e2e journeys remain a **hard gate before human testing** (a red staging never reaches the promote button) and the promote workflow must refuse to promote a commit whose staging e2e did not succeed. +- Secrets already exist and are reused: `REGISTRY_TOKEN` (also a Gitea API token), `PORTAINER_STAGING_WEBHOOK`, `PORTAINER_PROD_WEBHOOK`, `STAGING_BASE_URL`, `PROD_BASE_URL`. +- Staging quirk (memory): external `/api` is broken at the staging proxy — manual API testing happens from the host, not through stx.schoolcompare.co.uk; note it in the runbook, don't try to fix it in this plan. + +## Considered approaches (context for the reviewer) + +1. **Manual `workflow_dispatch` promote workflow (chosen).** Native on Gitea 1.26; the second approval is clicking "Run workflow" (or one API call) after testing staging. Least machinery, auditable via the Actions run history. +2. *Tag-driven promotion* (`push: tags: promote-*`): works on any Gitea version; approval = pushing a tag. Slightly more scriptable, less discoverable; kept as documented fallback only. +3. *GitOps `production` branch + promotion PR:* approval literally reuses the PR-review UI, but adds a second long-lived branch to keep in sync — too much ceremony for a solo project. Rejected. + +--- + +### Task 0: Branch + +- [ ] `git checkout main && git pull && git checkout -b chore/staged-prod-promotion` + +--- + +### Task 1: Stop the push-to-main workflow after the e2e gate + +**Files:** +- Modify: `.gitea/workflows/deploy.yml` + +**Interfaces:** +- Produces: images tagged `:sha-` + `:staging` (unchanged), a green `E2E Journeys against Staging` commit status that Task 2's promote workflow checks by name. **Do not rename the `e2e-staging` job's `name:` without updating Task 2's status check.** + +- [ ] **Step 1: Remove the auto-promotion** + +In `.gitea/workflows/deploy.yml`: +1. Change line 1 to: `name: Stage (build -> staging -> E2E gate)` +2. Delete the entire `promote-prod` job (lines 196–240 in the current file: from ` promote-prod:` to the end of the file). +3. Leave `build-*`, `deploy-staging`, and `e2e-staging` untouched. + +- [ ] **Step 2: Sanity-check the YAML** + +Run: `python3 -c "import yaml; yaml.safe_load(open('.gitea/workflows/deploy.yml')); print('yaml ok')"` +Expected: `yaml ok` + +- [ ] **Step 3: Commit** + +```bash +git add .gitea/workflows/deploy.yml +git commit -m "ci: stop deploy pipeline at staging; production promotion becomes manual" +``` + +--- + +### Task 2: Manual promote workflow + +**Files:** +- Create: `.gitea/workflows/promote.yml` + +**Interfaces:** +- Consumes: `:sha-` images built by deploy.yml; the `E2E Journeys against Staging` commit status. +- Produces: `:prod` and `:prod-previous` tags; prod stack update. + +- [ ] **Step 1: Write the workflow** + +```yaml +name: Promote to Production (manual) + +on: + workflow_dispatch: + inputs: + sha: + description: >- + Commit SHA on main to promote (full or >=7 chars). + Leave empty to promote the latest main commit. + required: false + default: "" + +env: + REGISTRY: privaterepo.sitaru.org + BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend + FRONTEND_IMAGE_NAME: ${{ gitea.repository }}-frontend + PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline + +jobs: + promote-prod: + name: Promote approved commit to Production + runs-on: ubuntu-latest + steps: + - name: Resolve target SHA + id: resolve + run: | + SHA_INPUT="${{ gitea.event.inputs.sha }}" + if [ -z "$SHA_INPUT" ]; then + SHA_INPUT="${{ gitea.sha }}" + fi + # Normalise to the full sha via the API so short inputs work + FULL_SHA=$(curl -fsS \ + -H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \ + "https://${REGISTRY}/api/v1/repos/${{ gitea.repository }}/git/commits/${SHA_INPUT}" \ + | python3 -c "import json,sys; print(json.load(sys.stdin)['sha'])") + SHORT_SHA="sha-$(echo "$FULL_SHA" | cut -c1-7)" + echo "full=$FULL_SHA" >> "$GITHUB_OUTPUT" + echo "short=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "Promoting $FULL_SHA (images tagged $SHORT_SHA)" + + - name: Verify the staging E2E gate passed for this commit + run: | + STATUS_JSON=$(curl -fsS \ + -H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \ + "https://${REGISTRY}/api/v1/repos/${{ gitea.repository }}/commits/${{ steps.resolve.outputs.full }}/status") + echo "$STATUS_JSON" | python3 -c " + import json, sys + d = json.load(sys.stdin) + ok = [s for s in d.get('statuses', []) + if 'E2E Journeys against Staging' in s.get('context', '') + and s.get('status') == 'success'] + if not ok: + print('REFUSED: no successful \"E2E Journeys against Staging\" status on this commit.') + print('Contexts found:', [s.get('context') for s in d.get('statuses', [])]) + sys.exit(1) + print('E2E gate verified green for this commit.') + " + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Gitea Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ gitea.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Retag approved images as prod (keeping rollback pointer) + run: | + SHORT_SHA="${{ steps.resolve.outputs.short }}" + for IMAGE in \ + "${REGISTRY}/${BACKEND_IMAGE_NAME}" \ + "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \ + "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do + docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true + docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}" + echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod" + done + + - name: Trigger production stack update + run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}" + + - name: Wait for production to become healthy + run: | + echo "Polling ${PROD_BASE_URL} for up to 5 minutes..." + for i in $(seq 1 60); do + if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then + echo "Production is up (attempt $i)" + exit 0 + fi + sleep 5 + done + echo "Production did not become healthy in time" >&2 + exit 1 + env: + PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }} +``` + +Implementation notes for the engineer: +- Gitea Actions uses the GitHub-compatible `$GITHUB_OUTPUT` file for step outputs; if the runner image doesn't populate it, fall back to `$GITEA_OUTPUT` (check the runner's docs/output at first run). +- The retag step is copied verbatim from the old `promote-prod` job except the SHA comes from the resolved input instead of `gitea.sha` — behaviour for the default (empty input on latest main) is identical to before. +- If `docker buildx imagetools create` fails with "not found" for `${IMAGE}:${SHORT_SHA}`, the chosen commit predates the registry's retention or never built — the error message is the desired behaviour (refuse loudly). + +- [ ] **Step 2: YAML sanity check** + +Run: `python3 -c "import yaml; yaml.safe_load(open('.gitea/workflows/promote.yml')); print('yaml ok')"` +Expected: `yaml ok` + +- [ ] **Step 3: Commit** + +```bash +git add .gitea/workflows/promote.yml +git commit -m "ci: manual production promotion workflow with e2e-gate verification" +``` + +--- + +### Task 3: Documentation — deploy model + runbook + +**Files:** +- Modify: `docs/DEPLOY.md` +- Modify: `claude.md` (the SDLC section) + +- [ ] **Step 1: Rewrite the flow description in `docs/DEPLOY.md`** + +Replace the staging→prod description with the new model (adapt to the file's existing structure; the substance to convey): + +```markdown +## Deploy model + +1. **PR → main (first approval).** Branch-protected merge; PR checks + (typecheck, tests, builds, AI review) must pass. +2. **Merge → staging (automatic).** Images are built once and tagged + `sha-` + `staging`; the staging stack updates; Playwright + journeys in `e2e/` run against staging. A red e2e run means staging + is not fit for testing — fix forward before considering promotion. +3. **Manual testing on staging.** stx.schoolcompare.co.uk. Note: + external `/api` is broken at the staging proxy — exercise API + endpoints from the host. +4. **Promote → production (second approval).** Actions → "Promote to + Production (manual)" → Run workflow. Leave the SHA empty to promote + the latest main, or paste a specific commit SHA. The workflow + refuses commits whose staging e2e gate is not green, retags the + images `:prod` (keeping `:prod-previous`), and updates the prod + stack. + +### Promotion granularity + +Staging always runs the latest `main`. Promoting approves a *state of +main*, not a single PR — if two PRs merged since the last promotion, +they ship together. Test staging accordingly. + +### Rollback + +Re-run "Promote to Production (manual)" with the SHA of the last good +commit (or retag manually: `docker buildx imagetools create -t +:prod :prod-previous` for each of the three images, then +POST the prod Portainer webhook). +``` + +- [ ] **Step 2: Update the SDLC bullet in `claude.md`** + +Replace the sentence "Merging to `main` deploys automatically: … retagged `:prod` and rolled out to production." with: + +```markdown +- Merging to `main` deploys automatically **to staging only**: images + are built once, deployed to the staging Portainer stack, and verified + by the Playwright journeys in `e2e/`. Production is a second, manual + approval: the "Promote to Production (manual)" workflow in Gitea + Actions, run after testing the feature on staging. It refuses commits + whose staging e2e gate isn't green. +``` + +- [ ] **Step 3: Commit** + +```bash +git add docs/DEPLOY.md claude.md +git commit -m "docs: two-stage deploy model (staging auto, production manual)" +``` + +--- + +### Task 4: PR + live validation + +- [ ] **Step 1: Push and open the PR** (Gitea API with credential-helper basic auth, as usual). PR body: the new model in three lines, the rollback recipe, and a warning that between merging this PR and its first promotion run, production receives no deployments (expected). + +- [ ] **Step 2: Validate after merge (human-in-the-loop):** +1. Merge this PR → confirm the `Stage (build -> staging -> E2E gate)` run goes green and **no** production deployment happens (prod image digest unchanged: `docker buildx imagetools inspect :prod` before/after, or check the Portainer prod stack's last-update time). +2. Test something trivial on staging. +3. Run "Promote to Production (manual)" with the SHA empty → confirm e2e verification passes, retag happens, prod becomes healthy. +4. Negative test: run the promote workflow with a garbage SHA (e.g. `deadbeef1`) → confirm it fails at resolve/verify without touching `:prod`. + +- [ ] **Step 3: Update the ledger/memory** with the new deploy model so future sessions stop assuming auto-promotion. + +--- + +## Out of scope / future options + +- Notifications when staging is ready for testing (Gitea can email on workflow completion; a webhook to ntfy/Matrix could be added later). +- Restricting who can run the promote workflow: Gitea 1.26 runs `workflow_dispatch` with the permissions of the dispatching user; for a solo repo this is already effectively restricted. +- The tag-driven fallback (`on: push: tags: promote-*`) if `workflow_dispatch` ever proves unreliable on the runner. diff --git a/docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md b/docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md new file mode 100644 index 0000000..d67821d --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-compare-screen-expert-review.md @@ -0,0 +1,177 @@ +# Compare Screen Redesign — Expert Data Review + +**Date:** 2026-07-11 +**Reviewer:** subagent briefed as an English education-standards / DfE-Ofsted data expert +**Subject:** desktop + mobile compare mockups and the redesign spec +(`2026-07-11-compare-screen-redesign-design.md`) +**Status:** first-pass must-fixes applied 2026-07-12; second-pass +findings (below) applied 2026-07-12 — mockups + spec §4/§8 updated + +## Must-fix + +1. **COVID gap is wrong and drops a real results year.** KS2 tests were + cancelled 2019/20 and 2020/21 only; they resumed in 2021/22 with + published school-level results (England RWM ≈ 59%). The mockup charts + omit 2021/22 entirely and the tooltip claims no tests were held + 2019/20–2021/22. Fix: add 2021/22 to axis and all series; shrink the + gap band; optionally annotate 2021/22 with DfE's post-pandemic + comparability caution. +2. **Report-card at-a-glance summary miscounts areas.** Detail list has + 4 Strong / 2 Expected / 1 Attention needed + Safeguarding met, but + the summary says "3 areas Expected standard" — it counts safeguarding + as a graded area. Safeguarding is a separate binary judgement and + must be excluded from rating counts. +3. **"Where the offers went" derivation is unsound.** Places − 1st-pref + offers ≠ "second or third choices": the residual can include 4th–6th + preference offers (pan-London scheme) and LA-allocated children who + didn't choose the school; and offers don't necessarily equal PAN. + Use the real 2nd/3rd-preference fields being promoted from + `raw.ees_admissions`; until then drop the row. +4. **Ofsted timeline in the copy is wrong.** Overall grades were + abolished September 2024, not November 2025; Sept 2024–Nov 2025 + inspections kept the four key judgements without an overall grade + (ungraded inspections carried grades forward). Neither mockup shows + the interim regime, which will dominate real comparisons. Fix copy + and add an interim example. +5. **Barclay's "published an overall grade only — no area-by-area + detail" misdescribes inspections.** No inspection type does that; a + 2021 graded inspection necessarily had subgrades — the gap is in our + dataset. If it was an ungraded (s8) inspection, "Outstanding" is a + carried-forward grade and should say so. Fix: "We don't hold + area-by-area detail for this inspection", and distinguish graded vs + ungraded in the data model. + +## Should-fix + +6. Writing is teacher assessment, not a test — "national tests and + teacher assessments"; note TA caveat on the Writing strip. +7. Verify renewed-framework wording against Ofsted's final toolkit: + likely "Needs attention" (not "Attention needed") and "Personal + development and well-being" (which otherwise collides with the + identically-named legacy judgement). Pin every label to the + published toolkit. +8. "Expected standard" now means two things on one page (Ofsted area + rating vs KS2 measure) — disambiguate in tooltips. +9. Disadvantaged row: DfE definition includes looked-after / previously + looked-after children, not just FSM6; benchmark labels inconsistent + across desktop/mobile; subgroup percentages need cohort sizes or a + volatility threshold before chips are attached. +10. "Trend, last 7 years" spans ten years; sparklines render the COVID + gap as equal spacing (the exact defect the audit criticises) and + "Improved: 52% → 87%" endpoint-cherry-picks a volatile series. +11. At-a-glance "Getting a place" uses different metrics per school + (Barclay is also oversubscribed on total preferences but shows a + green chip). Standardise on first-preference success %. Explain the + equal-preference rule; condition "living close by matters" on the + school's actual oversubscription criteria. +12. "457 applications for 180 places" = total preferences at any rank, + not head-to-head applicants; lead with first preferences vs places. + Add offers-vs-final-intake (waiting lists/appeals) caveat. +13. Elmhurst's subgrade list is likely missing Early years provision + (school has a nursery) — possible pipeline gap. +14. "Ofsted rating" label is obsolete post-Sept-2024 — use "Latest + Ofsted inspection"; check whether Oct 2021 is the latest inspection + or merely the latest graded one. +15. SEN: "EHCP plans" is redundant; 28% SEN support often indicates + resourced provision — add a note; England SEN-support ≈ 14%, not 13%. + +## Nice-to-have + +16. Consistent labelling of official DfE vs dataset-computed benchmarks + (and medians shouldn't be called averages inconsistently). +17. England 2015/16 RWM (53%) exists in DfE publications — the null is + a dataset gap; source it or the England line looks broken. +18. "1 in 4 first choices missed out" — actually more than 1 in 4. +19. "1,273 of 1,260 places (full)" is over capacity; capacity figures + are often stale — say "at or above capacity". +20. State the actual suppression rule (DfE: ≤5 pupils suppressed, + small numbers rounded) instead of "a handful". +21. Spec §4.3 progress chips can't exist for displayed years: KS2 + progress ended with 2022/23 (no KS1 baseline) and returns + ~2027/28 with the reception baseline. Make explicit in the spec. + IDACI (spec §4.5) is absent from mockups; if shipped, caveat it + describes pupils' neighbourhoods, not the school. +22. Tooltips should give the official term "first preference" alongside + the plain-English "first choice". + +## Overall assessment (verbatim gist) + +The bones are genuinely good by education-data standards — +England-average anchoring, explicit non-comparability messaging across +Ofsted regimes, refusal to synthesise an overall grade, time-true +x-axis, neutral FSM/EAL framing — better than most commercial +school-comparison sites. But items 1–5 are outright factual errors or +misdescriptions that a well-informed parent or Ofsted would catch; +the admissions section needs the most conceptual work (equal +preference, preferences-vs-applicants, offers-vs-intake). Fix 1–5 +before user testing; the rest fold into the planned PRs. + +--- + +# Second-pass review (2026-07-12) + +Same reviewer, after the must-fixes and the new three-tier metric +exposure model were applied. + +## Verification of first-pass must-fixes + +- **1 (COVID/2021/22): resolved.** Time-true axis, band covers only the + cancelled years, England 58.7% consistent with official figures, + dataset gaps break lines honestly; reading/maths England series all + match published figures; RWM ≤ min(subject) checks pass. +- **2 (report-card count): resolved** — safeguarding excluded, spec §8.2. +- **3 (offers derivation): resolved** — row removed, spec §8.3 bans it. +- **4 (Ofsted timeline): resolved on desktop; mobile omits the interim + regime clause** (see finding 6). +- **5 (Barclay explanation): resolved.** + +## New findings + +1. **Should-fix — scaled-score strip domain contradicts caption.** + Caption says "scaled scores run 80–120", strips render 100–120; + truncated domain exaggerates small gaps and below-100 averages + would fall off the edge. Render 80–120, or caption the 100–120 + window honestly and define below-100 behaviour. +2. **Should-fix — scaled-score England ticks (106/105/105) unsourced.** + Plausible but hand-entered; verify against DfE 2024/25 tables and + add loading official England scaled scores to the pipeline list + (absent from §8.1/§8.6). +3. **Should-fix — "Writing" listed under "Higher standard" in the + picker.** Writing TA outcome is "greater depth" (GDS), never + "higher standard". Label "Writing — greater depth (teacher + assessment)"; tooltip the combined higher-standard composition. +4. Nice — "grammar & punctuation" summary line drops "spelling" (GPS). +5. Nice — science is teacher-assessed (no KS2 test since 2009) and + coarse; tooltip it like writing; reconsider its tier-2 slot. +6. **Should-fix — mobile Ofsted copy skips the interim regime** + (Sept 2024–Nov 2025) that desktop explains. One clause fixes it. +7. **Should-fix — benchmark provenance still inconsistent** (EAL + tooltip unsourced; FSM/disadvantaged chips vs tooltips use three + vocabularies; header note says all England averages are official). + Adopt one house style: official = "England average", computed = + "benchmark / typical state school (our dataset)". Also tighten EAL + definition to census wording ("first language known or believed to + be other than English"). +8. Nice — "community primaries" distance note attached to an academy + (Elmhurst); say "non-faith primaries" or condition on policy field. +9. Nice — "Improving since 2022" → "since 2022/23". +10. Nice — England chart tooltips show decimals; §7 mandates whole + percents. + +## Residual gaps not covered by spec §8 + +11. Spec promises IDACI-in-words, Attendance section, and tier-2 + gender/absence that the mockups never show — mark post-v1 or + demonstrate, so implementation scope is unambiguous. +12. Add official England scaled-score averages to the pipeline task + list. +13. Add the writing/greater-depth terminology rule to §8.7. + +## Verdict + +All must-fixes genuinely resolved; the tier model is conceptually +sound ("no measure is lost", honest dataset-gap breaks, grouped +picker). Remaining issues are contained: one internal contradiction +(80–120 vs 100–120), one provenance inconsistency, one terminology +error (writing/GDS). With findings 1–3 and 6–7 addressed, the data +framing is fit to put in front of parents. diff --git a/docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md b/docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md new file mode 100644 index 0000000..b5778e2 --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md @@ -0,0 +1,324 @@ +# Compare Screen Redesign — Audit & Design + +**Date:** 2026-07-11 +**Status:** Draft — awaiting review +**Scope:** `/compare` page (nextjs-app), `/api/compare` endpoint (backend) + +## 1. Audit of the current screen + +The current compare page (`nextjs-app/components/ComparisonView.tsx`) is a +single-metric analyst tool: a ` + + + + + + + + + + + + + + + + + + + + + + + + School lines break where a year isn't in our dataset. + + +
+ +
+ + +

+ Sources: DfE Compare School Performance (KS2 results), Ofsted inspection outcomes, DfE school admissions data, school census — all from datasets SchoolCompare already collects. England averages for test results are the official DfE national figures; benchmarks for free school meals, language, SEN, school size and disadvantaged pupils' results are computed across all state schools in our dataset. Following DfE practice, figures based on 5 or fewer pupils are suppressed and shown as "no data". This is a static mockup: tooltips and "Add school" are illustrative, and Plumcroft's Ofsted report card is a made-up example of the November 2025 format (its real latest inspection is Good, June 2023) — no school in our dataset has a report card yet. +

+ + + diff --git a/docs/superpowers/specs/mockups/compare-mobile.html b/docs/superpowers/specs/mockups/compare-mobile.html new file mode 100644 index 0000000..fbb73f1 --- /dev/null +++ b/docs/superpowers/specs/mockups/compare-mobile.html @@ -0,0 +1,437 @@ +Compare screen — mobile mockup + + +
+

Mobile mockup — proposed /compare. Mobile-first layout: measures stack vertically with all schools under each, so nothing needs horizontal swiping. Same live data as the desktop mockup.

+ +

Compare schools

+

Anchored against the England average — the grey tick — so you can tell what's typical at a glance.

+ +
+ Barclay + Elmhurst + Plumcroft + + Add +
+ +

At a glance

+

The short version — each measure is explained in its own section below.

+ +
+
Latest Ofsted inspection
+
BarclayOutstandingOlder-style inspection, Oct 2021
+
ElmhurstOutstandingOlder-style inspection, Oct 2021
+
Plumcroft4 areas Strong standard 2 areas Expected Attendance & behaviour: Attention needed illustrativeNew-style report card, Nov 2025 · safeguarding met · full detail in the Ofsted section below
+
+ +
+
Children reaching the expected standard ?
+
England average: 62%
+
Barclay87% Above average
+
Elmhurst92% Above average
+
Plumcroft79% Above average
+
+ +
+
Getting a place
+
Barclay97% of first choices offeredNamed on 457 forms · 180 places
+
Elmhurst73% of first choices offeredNamed on 342 forms · 120 places
+
PlumcroftAll first choices offeredNamed on 185 forms · 80 places
+
+ +

Ofsted inspection

+

Ofsted stopped giving a single overall grade in September 2024 (inspections until November 2025 kept the area-by-area judgements); from November 2025 new inspections produce a report card rating each area of school life (Exceptional · Strong standard · Expected standard · Attention needed · Urgent improvement). A report card and an older grade aren't directly comparable. Ofsted's "Expected standard" rating is unrelated to the KS2 test measure below.

+ +
+
Latest inspection
+
BarclayOutstanding4+ years ago7 Oct 2021 · we don't hold area-by-area detail for this inspection · Ofsted page →
+
ElmhurstOutstanding 4+ years ago 6 Oct 2021 +
+
Quality of educationOutstanding
+
Behaviour & attitudesOutstanding
+
Personal developmentOutstanding
+
Leadership & managementOutstanding
+
+ Ofsted page → +
+
PlumcroftReport card illustrative 14 Nov 2025 +
+
AchievementStrong standard
+
Curriculum & teachingStrong standard
+
Attendance & behaviourAttention needed
+
Personal developmentStrong standard
+
InclusionExpected standard
+
Leadership & governanceStrong standard
+
Early yearsExpected standard
+
SafeguardingMet
+
+ Ofsted page → +
+
+ +

How children do academically

+

End of Year 6 national tests and teacher assessments (2024/25) — writing is teacher-assessed. Each line runs 0–100%; the grey tick is the England average.

+
+
+ More measures — grammar, punctuation & spelling, science, scaled scores +
+

Strips show the 100–120 window of the full 80–120 scaled-score range; 100 is the expected standard (the strip widens if a school averages below it). England ticks for GPS and science aren't in our dataset yet, and the scaled-score ticks are indicative — official DfE figures will be loaded before launch.

+
+
+ +
+
Children from lower-income families ?
+
State-school average: 46%
+
Barclay86% Well above average
+
Elmhurst93% Well above average
+
Plumcroft72% Above average
+
+ +

Getting a place

+

September 2026 entry. "First choice" = families who ranked the school top of their form (officially a "first preference"). Schools never see your ranking — places go by the admission criteria alone. Figures are National Offer Day offers; waiting lists and appeals can change the final intake.

+
+
First-choice families offered a place
+
Barclay97%Named on 457 forms · 180 places
+
Elmhurst73% Over 1 in 4 missed outNamed on 342 forms · 120 places — check the school's admission criteria (for most non-faith primaries, distance decides)
+
Plumcroft100%Named on 185 forms · 80 places · every first choice offered
+
+ +

Who goes there

+

From the latest school census (2025/26). No "right" numbers here — just context.

+
+
Pupils on roll
+
Barclay1,273At or above capacity · much larger than average · girls 51% / boys 49%
+
Elmhurst98098% full · much larger than average · girls 48% / boys 52%
+
Plumcroft1,056At or above capacity · much larger than average · girls 51% / boys 49%
+
+
+
Free school meals ?
+
State-school average: 25% (our dataset)
+
Barclay26% About average
+
Elmhurst25% About average
+
Plumcroft30% A little above
+
+
+
English as an additional language · extra learning support (SEN) ?
+
BarclayEAL 62% · SEN 6%
+
ElmhurstEAL 84% · SEN 8%
+
PlumcroftEAL 20% · SEN 28% SEN well above avg
+
+
+
Basics
+
BarclayAges 3–11 · nursery · no faith · Lion Academy Trust
+
ElmhurstAges 3–11 · nursery · no faith · New Vision Trust
+
PlumcroftAges 3–11 · nursery · no faith · Greenwich council
+
+ +

Explore trends

+

Every measure from the current compare page lives on here, grouped. Three are wired up in this mockup. School lines break where a year isn't in our dataset.

+
+ +
+
+ +
+
+

← swipe the chart →

+ +

+ Sources: DfE Compare School Performance, Ofsted inspection outcomes, DfE admissions data, school census — all from datasets SchoolCompare already collects. England averages for test results are official DfE figures; FSM, language, SEN, size and disadvantaged-pupil benchmarks are computed across state schools in our dataset. Plumcroft's Ofsted report card is a made-up example of the November 2025 format (its real latest inspection is Good, June 2023). Following DfE practice, figures based on 5 or fewer pupils are suppressed and shown as "no data". Static mockup — tooltips and "+ Add" are illustrative. +

+
+ + diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index cf68899..6ae1d16 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -19,6 +19,27 @@ function schoolLinks(page: Page) { return page.locator('a[href^="/school/"]'); } +/** + * Two URNs guaranteed to be pure-primary (same phase). The compare page's + * phase tabs split all-through schools (which carry KS4 data) onto the + * secondary tab, so picking two arbitrary "primary" search hits can land + * them on different tabs where only the active one renders. Selecting via + * the API by exact phase keeps both on the same tab. Data-invariant: uses + * whatever primaries the environment holds. + */ +async function twoPrimaryUrns(page: Page): Promise<[string, string]> { + const res = await page.request.get('/api/schools?search=primary&per_page=50'); + expect(res.ok()).toBeTruthy(); + const body = await res.json(); + const urns: string[] = (body.schools ?? []) + .filter((s: { phase?: string; rwm_expected_pct?: number | null }) => + s.phase === 'Primary' && s.rwm_expected_pct != null, + ) + .map((s: { urn: number }) => String(s.urn)); + expect(urns.length).toBeGreaterThanOrEqual(2); + return [urns[0], urns[1]]; +} + test('home page loads with hero search', async ({ page }) => { await page.goto('/'); await expect(page.locator('h1').first()).toBeVisible(); @@ -138,20 +159,44 @@ test('results map fullscreen falls back to an overlay on iOS', async ({ page }) await expect(openFs).toBeVisible(); }); -test('comparing two schools shows both side by side', async ({ page }) => { - // Collect two school URNs from search results, then load the share URL - await searchByName(page, 'primary'); - await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 }); - const hrefs = await schoolLinks(page).evaluateAll((links) => - links.map((l) => (l as HTMLAnchorElement).getAttribute('href') || '') - ); - const urns = [...new Set(hrefs.map((h) => h.match(/\/school\/(\d+)/)?.[1]).filter(Boolean))]; - expect(urns.length).toBeGreaterThanOrEqual(2); +test('comparing two schools shows the parent-first sections side by side', async ({ page }) => { + // Two same-phase (pure primary) schools so both stay on one tab. + const [urn0, urn1] = await twoPrimaryUrns(page); - await page.goto(`/compare?urns=${urns[0]},${urns[1]}`); + await page.goto(`/compare?urns=${urn0},${urn1}`); // Both schools' detail links should render in the comparison view - await expect(page.locator(`a[href*="${urns[0]}"]`).first()).toBeVisible({ timeout: 15_000 }); - await expect(page.locator(`a[href*="${urns[1]}"]`).first()).toBeVisible(); + await expect(page.locator(`a[href*="${urn0}"]`).first()).toBeVisible({ timeout: 15_000 }); + await expect(page.locator(`a[href*="${urn1}"]`).first()).toBeVisible(); + + // The parent-first sections render in order (data-invariant: headings only) + for (const heading of [ + 'At a glance', + 'Ofsted inspection', + /How (children|students) do academically/, + 'Who goes there', + 'Explore trends', + ]) { + await expect( + page.getByRole('heading', { name: heading }).first(), + ).toBeVisible({ timeout: 15_000 }); + } + + // Every number gets an anchor: at least one England-average tick or label + await expect(page.getByText(/England \d+/).first()).toBeVisible(); + + // Ofsted linkout goes to the school's provider page, never a report deep-link + const ofstedLink = page.getByRole('link', { name: /Ofsted page/i }).first(); + await expect(ofstedLink).toBeVisible(); + expect(await ofstedLink.getAttribute('href')).toMatch( + /reports\.ofsted\.gov\.uk\/provider\/21\/\d+/ + ); + + // A school never shows both an overall-grade badge AND report-card detail: + // "Report card" implies "no overall grade is given" copy is present too. + const reportCards = await page.getByText('Report card', { exact: true }).count(); + if (reportCards > 0) { + await expect(page.getByText(/no overall grade/i).first()).toBeVisible(); + } }); test('compare chart on mobile shows school chips with tap-to-focus', async ({ page }) => { @@ -170,9 +215,40 @@ test('compare chart on mobile shows school chips with tap-to-focus', async ({ pa expect(urns.length).toBeGreaterThanOrEqual(3); await page.goto(`/compare?urns=${urns[0]},${urns[1]},${urns[2]}`); - await expect(page.locator('canvas:visible').first()).toBeVisible({ timeout: 15_000 }); - // The mobile chart legend renders one chip per school in the active phase. + // Mobile is measure-first: the At a glance section stacks all active-phase + // schools inside one flow — no horizontal swiping between school columns. + await expect( + page.getByRole('heading', { name: 'At a glance' }), + ).toBeVisible({ timeout: 15_000 }); + const body = page.locator('body'); + const bodyOverflowsX = await body.evaluate( + (el) => el.scrollWidth > el.clientWidth + 1, + ); + expect(bodyOverflowsX).toBe(false); + + // The sticky school bar must pin *below* the sticky site header, not at + // top:0 where the header covers it and the selected schools are hidden. + // Assert the sticky offset directly (robust — no scroll timing needed). + const barTop = await page + .locator('[class*="schoolBar"]') + .first() + .evaluate((el) => parseFloat(getComputedStyle(el).top)); + const headerHeight = await page + .locator('[class*="header"]') + .first() + .evaluate((el) => el.getBoundingClientRect().height); + expect(barTop).toBeGreaterThanOrEqual(headerHeight - 1); + + // The trends chart still renders (inside the Explore trends section)… + const chartCanvas = page.locator('canvas:visible').first(); + await expect(chartCanvas).toBeVisible({ timeout: 15_000 }); + // …at a real height, not the squashed ~150px Chart.js fallback that + // appears when the container lacks a definite height. + const chartBox = await chartCanvas.boundingBox(); + expect(chartBox && chartBox.height).toBeGreaterThan(220); + + // …with the mobile chart legend chips and tap-to-focus behaviour intact. const chipGroup = page.getByRole('group', { name: /highlight a school/i }); const chips = chipGroup.getByRole('button'); await expect(chips.first()).toBeVisible({ timeout: 15_000 }); diff --git a/nextjs-app/__tests__/components/CompareOfsted.test.tsx b/nextjs-app/__tests__/components/CompareOfsted.test.tsx new file mode 100644 index 0000000..849d255 --- /dev/null +++ b/nextjs-app/__tests__/components/CompareOfsted.test.tsx @@ -0,0 +1,106 @@ +import { render, screen } from '@testing-library/react'; + +import { CompareOfsted } from '@/components/compare/CompareOfsted'; +import type { ComparisonData, OfstedInspection, School } from '@/lib/types'; + +function school(urn: number, name: string): School { + return { urn, school_name: name } as School; +} + +function ofsted(partial: Partial): OfstedInspection { + return { + framework: null, + inspection_date: '2021-10-07', + inspection_type: null, + overall_effectiveness: null, + quality_of_education: null, + behaviour_attitudes: null, + personal_development: null, + leadership_management: null, + early_years_provision: null, + previous_overall: null, + rc_safeguarding_met: null, + rc_inclusion: null, + rc_curriculum_teaching: null, + rc_achievement: null, + rc_attendance_behaviour: null, + rc_personal_development: null, + rc_leadership_governance: null, + rc_early_years: null, + rc_sixth_form: null, + ofsted_page_url: 'https://reports.ofsted.gov.uk/provider/21/1', + ...partial, + }; +} + +const schools = [school(1, 'Graded School'), school(2, 'Carried School'), school(3, 'Card School')]; + +const data: Record = { + '1': { + school_info: schools[0], + yearly_data: [], + ofsted: ofsted({ overall_effectiveness: 1, grade_source: 'graded' }), + }, + '2': { + school_info: schools[1], + yearly_data: [], + ofsted: ofsted({ overall_effectiveness: 2, grade_source: 'ungraded_carried_forward' }), + }, + '3': { + school_info: schools[2], + yearly_data: [], + ofsted: ofsted({ + inspection_date: '2025-11-14', + rc_safeguarding_met: true, + report_card: { + rc_achievement: { code: 2, label: 'Strong standard' }, + rc_attendance_behaviour: { code: 4, label: 'Needs attention' }, + }, + }), + }, +}; + +describe('CompareOfsted', () => { + it('renders the three regimes without inventing an overall grade for report cards', () => { + render(); + + expect(screen.getByText('Outstanding')).toBeInTheDocument(); + // Carried-forward grade is shown but marked as such + expect(screen.getByText('Good')).toBeInTheDocument(); + expect(screen.getByText(/carried forward/i)).toBeInTheDocument(); + // Report card: label present, no overall-grade badge for that school + expect(screen.getByText('Report card')).toBeInTheDocument(); + expect(screen.getByText(/no overall grade/i)).toBeInTheDocument(); + }); + + it('uses one chip-list grammar for both regimes in judgement detail', () => { + render(); + // report-card area chip + expect(screen.getByText('Attendance & behaviour')).toBeInTheDocument(); + expect(screen.getByText('Needs attention')).toBeInTheDocument(); + // graded school without published subgrades → honest dataset statement + expect( + screen.getAllByText(/We don't hold area-by-area detail/i).length, + ).toBeGreaterThanOrEqual(1); + }); + + it('shows the mixed-regime comparability note only when regimes differ', () => { + render(); + expect(screen.getByText(/aren't directly comparable/i)).toBeInTheDocument(); + }); + + it('links every school to its Ofsted page', () => { + render(); + const links = screen.getAllByRole('link', { name: /Ofsted page/i }); + expect(links).toHaveLength(3); + expect(links[0]).toHaveAttribute('href', 'https://reports.ofsted.gov.uk/provider/21/1'); + }); + + it('renders a per-measure mobile tag with the short school name', () => { + render(); + // Each measure repeats the schools, so the short name ("Graded" from + // "Graded School") appears once per measure (4) via the cell tag. + expect(screen.getAllByText('Graded').length).toBe(4); + expect(screen.getAllByText('Card').length).toBe(4); + }); +}); diff --git a/nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx b/nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx new file mode 100644 index 0000000..90bdae9 --- /dev/null +++ b/nextjs-app/__tests__/components/ComparisonView.refresh.test.tsx @@ -0,0 +1,82 @@ +/** + * Regression: on refresh, the compare page must show the SSR-rendered data. + * + * The basket hydrates from the URL a beat after mount (selectedSchools is + * empty for the first render), so the fetch effect must not blank the + * SSR payload during that window — and must not refetch data the server + * already provided. + */ + +import { render, screen, waitFor } from '@testing-library/react'; + +import { ComparisonView } from '@/components/ComparisonView'; +import { ComparisonProvider } from '@/context/ComparisonProvider'; +import type { ComparisonData, School } from '@/lib/types'; + +const fetchComparison = jest.fn(); +jest.mock('@/lib/api', () => ({ + fetchComparison: (...args: unknown[]) => fetchComparison(...args), +})); +jest.mock('@/lib/analytics', () => ({ track: jest.fn() })); + +function school(urn: number, name: string): School { + return { + urn, + school_name: name, + local_authority: 'Testshire', + school_type: 'Community school', + rwm_expected_pct: 80, + phase: 'Primary', + } as School; +} + +function data(urn: number, name: string): ComparisonData { + return { + school_info: school(urn, name), + yearly_data: [{ year: 202425, rwm_expected_pct: 80 }] as ComparisonData['yearly_data'], + ofsted: null, + census: null, + admissions: null, + admissions_history: [], + deprivation: null, + }; +} + +const INITIAL_DATA = { + '100': data(100, 'Alpha Primary'), + '200': data(200, 'Beta Primary'), +}; + +beforeEach(() => { + fetchComparison.mockReset(); +}); + +test('renders SSR data on refresh without wiping it or refetching', async () => { + render( + + + , + ); + + // Both SSR-provided schools appear (data was not blanked during hydration) + await waitFor(() => { + expect(screen.getAllByText('Alpha Primary').length).toBeGreaterThan(0); + }); + expect(screen.getAllByText('Beta Primary').length).toBeGreaterThan(0); + expect(screen.getByRole('heading', { name: 'At a glance' })).toBeInTheDocument(); + + // …and the client never refetched data the server already rendered. + expect(fetchComparison).not.toHaveBeenCalled(); +}); diff --git a/nextjs-app/__tests__/components/DotStrip.test.tsx b/nextjs-app/__tests__/components/DotStrip.test.tsx new file mode 100644 index 0000000..786f183 --- /dev/null +++ b/nextjs-app/__tests__/components/DotStrip.test.tsx @@ -0,0 +1,48 @@ +import { render, screen } from '@testing-library/react'; + +import { DotStrip } from '@/components/DotStrip'; + +describe('DotStrip', () => { + it('enumerates anchor and school values in the aria-label', () => { + render( + , + ); + const strip = screen.getByRole('img'); + expect(strip).toHaveAccessibleName( + 'Reading: England 75%, Barclay 91%, Elmhurst 92%, Plumcroft 87%', + ); + }); + + it('renders the anchor tick when provided and not otherwise', () => { + const { rerender } = render( + , + ); + expect(screen.getByText('England 75%')).toBeInTheDocument(); + + rerender( + , + ); + expect(screen.queryByText(/England/)).not.toBeInTheDocument(); + }); + + it('skips schools without a value', () => { + render( + , + ); + expect(screen.getByRole('img')).toHaveAccessibleName('Maths: Barclay 91%'); + }); +}); diff --git a/nextjs-app/__tests__/lib/compareChartData.test.ts b/nextjs-app/__tests__/lib/compareChartData.test.ts new file mode 100644 index 0000000..09325ae --- /dev/null +++ b/nextjs-app/__tests__/lib/compareChartData.test.ts @@ -0,0 +1,84 @@ +/** + * buildCompareChart: every selected school must produce a rendered series + * (regression guard for the production bug where a third school's line + * vanished), the x-axis must include cancelled/unpublished years as real + * gaps (never compressing time), and the England overlay renders dashed + * with no gap-bridging. + */ + +import { buildCompareChart, fillAcademicYears } from '@/lib/compareChartData'; +import type { ComparisonData } from '@/lib/types'; + +function school(urn: number, years: Array<[number, number | null]>): ComparisonData { + return { + school_info: { urn, school_name: `School ${urn}` } as ComparisonData['school_info'], + yearly_data: years.map(([year, v]) => ({ year, rwm_expected_pct: v })) as ComparisonData['yearly_data'], + }; +} + +const THREE_SCHOOLS = { + '1': school(1, [[201819, 87], [202223, 87], [202425, 87]]), + '2': school(2, [[201819, 88], [202223, 88], [202425, 92]]), + '3': school(3, [[201819, 69], [202223, 62], [202425, 79]]), +}; + +const SCHOOL_LIST = [1, 2, 3].map((urn) => ({ urn, school_name: `School ${urn}` })); + +describe('fillAcademicYears', () => { + it('fills every academic year between min and max', () => { + expect(fillAcademicYears([201819, 202223])).toEqual([ + 201819, 201920, 202021, 202122, 202223, + ]); + }); +}); + +describe('buildCompareChart', () => { + it('renders one series per selected school — none silently dropped', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(chart.schoolDatasets).toHaveLength(3); + for (const ds of chart.schoolDatasets) { + expect(ds.data.some((v) => v != null)).toBe(true); + } + }); + + it('handles float years from the API (202425.0 style)', () => { + const floaty = { + '1': school(1, [[201819.0 as number, 80], [202425.0 as number, 85]]), + }; + const chart = buildCompareChart(floaty, [SCHOOL_LIST[0]], 'rwm_expected_pct'); + expect(chart.schoolDatasets[0].data.filter((v) => v != null)).toHaveLength(2); + }); + + it('includes cancelled/unpublished years as null gaps, not compressed time', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(chart.years).toContain(201920); + expect(chart.years).toContain(202122); + const idx = chart.years.indexOf(202021); + expect(chart.schoolDatasets[0].data[idx]).toBeNull(); + }); + + it('adds a dashed England overlay when national data is supplied', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', { + 201819: 64.9, + 202122: 58.7, + 202223: 59.5, + 202425: 62.1, + }); + expect(chart.englandDataset).not.toBeNull(); + const eng = chart.englandDataset!; + expect(eng.label).toBe('England average'); + expect(eng.borderDash).toEqual([5, 4]); + expect(eng.spanGaps).toBe(false); + // England has a value for 2021/22 even though schools do not + expect(eng.data[chart.years.indexOf(202122)]).toBe(58.7); + }); + + it('flags the unpublished 2021/22 school-level year when England has data but schools do not', () => { + const withNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', { + 202122: 58.7, + }); + expect(withNational.showUnpublished202122Note).toBe(true); + const withoutNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(withoutNational.showUnpublished202122Note).toBe(false); + }); +}); diff --git a/nextjs-app/__tests__/lib/compareLogic.test.ts b/nextjs-app/__tests__/lib/compareLogic.test.ts new file mode 100644 index 0000000..e1308bf --- /dev/null +++ b/nextjs-app/__tests__/lib/compareLogic.test.ts @@ -0,0 +1,266 @@ +/** + * compareLogic encodes the expert-reviewed comprehension rules for the + * compare screen: report-card summarisation (safeguarding never counted), + * three-regime Ofsted display, one consistent admissions chip metric, + * CI-based progress banding, verdict chips and dot-strip geometry. + */ + +import { + OFSTED_LEGACY_GRADES, + ofstedDisplay, + progressBand, + rcAreaLabel, + stripPositions, + summariseAdmissions, + summariseReportCard, + verdict, +} from '@/lib/compareLogic'; +import type { OfstedInspection, SchoolAdmissions } from '@/lib/types'; + +function ofsted(partial: Partial): OfstedInspection { + return { + framework: null, + inspection_date: null, + inspection_type: null, + overall_effectiveness: null, + quality_of_education: null, + behaviour_attitudes: null, + personal_development: null, + leadership_management: null, + early_years_provision: null, + previous_overall: null, + rc_safeguarding_met: null, + rc_inclusion: null, + rc_curriculum_teaching: null, + rc_achievement: null, + rc_attendance_behaviour: null, + rc_personal_development: null, + rc_leadership_governance: null, + rc_early_years: null, + rc_sixth_form: null, + ...partial, + }; +} + +const REPORT_CARD = { + rc_achievement: { code: 2, label: 'Strong standard' }, + rc_curriculum_teaching: { code: 2, label: 'Strong standard' }, + rc_personal_development: { code: 2, label: 'Strong standard' }, + rc_leadership_governance: { code: 2, label: 'Strong standard' }, + rc_inclusion: { code: 3, label: 'Expected standard' }, + rc_early_years: { code: 3, label: 'Expected standard' }, + rc_attendance_behaviour: { code: 4, label: 'Needs attention' }, +}; + +describe('summariseReportCard', () => { + it('counts graded areas best-first and NAMES problem areas', () => { + const s = summariseReportCard( + ofsted({ report_card: REPORT_CARD, rc_safeguarding_met: true }), + ); + expect(s.counts).toEqual([ + { label: 'Strong standard', count: 4 }, + { label: 'Expected standard', count: 2 }, + ]); + expect(s.problems).toEqual([ + { areaLabel: 'Attendance & behaviour', label: 'Needs attention' }, + ]); + expect(s.safeguarding).toBe('met'); + expect(s.allClear).toBe(false); + }); + + it('never counts safeguarding as a graded area', () => { + const s = summariseReportCard( + ofsted({ + report_card: { rc_achievement: { code: 3, label: 'Expected standard' } }, + rc_safeguarding_met: true, + }), + ); + const total = s.counts.reduce((n, c) => n + c.count, 0); + expect(total).toBe(1); + }); + + it('is allClear when everything is Expected standard or better and safeguarding met', () => { + const s = summariseReportCard( + ofsted({ + report_card: { + rc_achievement: { code: 3, label: 'Expected standard' }, + rc_inclusion: { code: 1, label: 'Exceptional' }, + }, + rc_safeguarding_met: true, + }), + ); + expect(s.allClear).toBe(true); + expect(s.counts[0]).toEqual({ label: 'Exceptional', count: 1 }); + }); + + it('passes labels through from the API — never invents wording', () => { + const s = summariseReportCard( + ofsted({ report_card: { rc_inclusion: { code: 4, label: 'Needs attention' } } }), + ); + expect(JSON.stringify(s)).not.toContain('Attention needed'); + }); +}); + +describe('ofstedDisplay', () => { + it('prefers the report card over any legacy grade', () => { + const d = ofstedDisplay( + ofsted({ overall_effectiveness: 2, report_card: REPORT_CARD }), + ); + expect(d.kind).toBe('report_card'); + }); + + it('distinguishes graded from carried-forward grades', () => { + const graded = ofstedDisplay( + ofsted({ overall_effectiveness: 1, grade_source: 'graded' }), + ); + expect(graded).toMatchObject({ kind: 'graded', gradeLabel: 'Outstanding', carriedForward: false }); + + const carried = ofstedDisplay( + ofsted({ overall_effectiveness: 2, grade_source: 'ungraded_carried_forward' }), + ); + expect(carried).toMatchObject({ kind: 'carried_forward', gradeLabel: 'Good', carriedForward: true }); + }); + + it('handles missing data', () => { + expect(ofstedDisplay(null).kind).toBe('none'); + expect(ofstedDisplay(ofsted({})).kind).toBe('none'); + }); + + it('identifies transitional inspections without overall grades', () => { + const transitional = ofstedDisplay( + ofsted({ overall_effectiveness: null, inspection_date: '2024-11-05' }), + ); + expect(transitional.kind).toBe('transitional'); + }); + + it('uses the four legacy grade words', () => { + expect(OFSTED_LEGACY_GRADES).toEqual({ + 1: 'Outstanding', + 2: 'Good', + 3: 'Requires improvement', + 4: 'Inadequate', + }); + }); +}); + +describe('rcAreaLabel', () => { + it('maps rc keys to the mockups’ area labels', () => { + expect(rcAreaLabel('rc_attendance_behaviour')).toBe('Attendance & behaviour'); + expect(rcAreaLabel('rc_curriculum_teaching')).toBe('Curriculum & teaching'); + expect(rcAreaLabel('rc_leadership_governance')).toBe('Leadership & governance'); + }); +}); + +describe('summariseAdmissions', () => { + function admissions(partial: Partial): SchoolAdmissions { + return { + year: 202627, + places_offered: null, + total_applications: null, + first_preference_offer_pct: null, + oversubscribed: null, + ...partial, + }; + } + + it('97% → good chip with the mockup wording', () => { + const s = summariseAdmissions( + admissions({ first_preference_offer_pct: 96.98, total_applications: 457, places_offered: 180 }), + ); + expect(s.chip).toEqual({ tone: 'good', text: '97% of first choices offered' }); + expect(s.interest).toBe('Named on 457 forms · 180 places'); + }); + + it('73% → warn chip "Over 1 in 4 first choices missed out"', () => { + const s = summariseAdmissions(admissions({ first_preference_offer_pct: 73.4 })); + expect(s.chip).toEqual({ tone: 'warn', text: 'Over 1 in 4 first choices missed out' }); + }); + + it('100% → "All first choices offered"', () => { + const s = summariseAdmissions(admissions({ first_preference_offer_pct: 100 })); + expect(s.chip).toEqual({ tone: 'good', text: 'All first choices offered' }); + }); + + it('no data → null chip and interest', () => { + const s = summariseAdmissions(null); + expect(s.chip).toBeNull(); + expect(s.interest).toBeNull(); + }); +}); + +describe('progressBand', () => { + it('CI entirely above zero → above', () => { + expect(progressBand(1.2, 0.4, 2.0)).toBe('above'); + }); + it('CI entirely below zero → below', () => { + expect(progressBand(-1.2, -2.0, -0.4)).toBe('below'); + }); + it('CI straddling zero → average', () => { + expect(progressBand(0.3, -0.5, 1.1)).toBe('average'); + }); + it('missing CI → null (no naive thresholding)', () => { + expect(progressBand(1.2, null, null)).toBeNull(); + expect(progressBand(null, null, null)).toBeNull(); + }); +}); + +describe('verdict', () => { + it('above / close / below with a 2pp tolerance', () => { + expect(verdict(87, 62)).toBe('above'); + expect(verdict(61, 62)).toBe('close'); + expect(verdict(40, 62)).toBe('below'); + }); +}); + +describe('stripPositions', () => { + it('maps a custom domain', () => { + const pts = stripPositions([106], 100, 120); + expect(pts[0].pos).toBe(30); + }); + + it('flips a colliding label above', () => { + const pts = stripPositions([91, 92], 0, 100); + const sorted = [...pts].sort((a, b) => a.value - b.value); + expect(sorted[0].labelAbove).toBe(false); + expect(sorted[1].labelAbove).toBe(true); + }); + + it('skips nulls and keeps school indices', () => { + const pts = stripPositions([50, null, 70], 0, 100); + expect(pts).toHaveLength(2); + expect(pts.map((p) => p.schoolIndex)).toEqual([0, 2]); + }); + + it('clamps out-of-domain values', () => { + const pts = stripPositions([95], 100, 120); + expect(pts[0].pos).toBe(0); + }); +}); + +describe('latestValues', () => { + const data = { + '1': { + yearly_data: [ + { year: 202324, rwm_expected_pct: 75 }, + { year: 202425, rwm_expected_pct: 87 }, + ], + }, + '2': { + yearly_data: [ + { year: 202324, rwm_expected_pct: 82 }, + { year: 202425, rwm_expected_pct: null }, + ], + }, + }; + + it('takes the latest non-null value per school in urn order', async () => { + const { latestValues } = await import('@/lib/compareLogic'); + expect(latestValues(data, [1, 2], 'rwm_expected_pct')).toEqual([87, 82]); + }); + + it('returns null for unknown schools and metrics', async () => { + const { latestValues } = await import('@/lib/compareLogic'); + expect(latestValues(data, [3], 'rwm_expected_pct')).toEqual([null]); + expect(latestValues(data, [1], 'nope')).toEqual([null]); + }); +}); diff --git a/nextjs-app/__tests__/lib/utils.test.ts b/nextjs-app/__tests__/lib/utils.test.ts index a1ebbbc..9fb9fba 100644 --- a/nextjs-app/__tests__/lib/utils.test.ts +++ b/nextjs-app/__tests__/lib/utils.test.ts @@ -10,6 +10,7 @@ import { debounce, buildOfstedListBadge, metricKind, + shortName, computeYBounds, } from '@/lib/utils'; @@ -223,3 +224,17 @@ describe('isProposedToClose', () => { expect(isProposedToClose({})).toBe(false); }); }); + +describe('shortName', () => { + it('drops the trailing establishment-type words', () => { + expect(shortName('Barclay Primary School')).toBe('Barclay'); + expect(shortName('Elmhurst Primary School')).toBe('Elmhurst'); + expect(shortName("St Mary's Catholic Primary School")).toBe("St Mary's"); + expect(shortName('Riverside Community Junior School')).toBe('Riverside'); + }); + + it('keeps a name that carries no type suffix, capping very long ones', () => { + expect(shortName('Beaver Road')).toBe('Beaver Road'); + expect(shortName('A'.repeat(30), 10)).toBe('AAAAAAAAA…'); + }); +}); diff --git a/nextjs-app/app/compare/page.tsx b/nextjs-app/app/compare/page.tsx index 1a1ff38..0b2b4e4 100644 --- a/nextjs-app/app/compare/page.tsx +++ b/nextjs-app/app/compare/page.tsx @@ -16,8 +16,10 @@ interface ComparePageProps { export const metadata: Metadata = { title: 'Compare Schools', - description: 'Compare KS2 performance across multiple primary schools in England', - keywords: 'school comparison, compare schools, KS2 comparison, primary school performance', + description: + 'Compare schools in England side by side — Ofsted inspections, KS2 and GCSE results against the England average, admissions odds and school community.', + keywords: + 'school comparison, compare schools, Ofsted comparison, school admissions, KS2 comparison, primary school performance', }; // Dynamic via searchParams; remove force-dynamic so internal data fetches @@ -30,26 +32,24 @@ export default async function ComparePage({ searchParams }: ComparePageProps) { const selectedMetric = metricParam || 'rwm_expected_pct'; try { - // Fetch comparison data if URNs provided - let comparisonData = null; - if (urns.length > 0) { - try { - const response = await fetchComparison(urnsParam!); - comparisonData = response.comparison; - } catch (error) { - console.error('Failed to fetch comparison:', error); - } - } + // Fetch comparison + metrics in parallel — they are independent. + const [comparisonResponse, metricsResponse] = await Promise.all([ + urns.length > 0 + ? fetchComparison(urnsParam!).catch((error) => { + console.error('Failed to fetch comparison:', error); + return null; + }) + : Promise.resolve(null), + fetchMetrics(), + ]); - // Fetch available metrics - const metricsResponse = await fetchMetrics(); - - // Metrics is already an array const metricsArray = metricsResponse?.metrics || []; return ( ; metric: string; metricLabel: string; + /** Official England figure per academic year for this metric — renders a + * dashed grey reference line when provided. */ + nationalByYear?: Record; } // One shape per basket slot (MAX_SCHOOLS = 5) — secondary encoding so // converging lines stay tellable apart without relying on hue alone. const POINT_STYLES: PointStyle[] = ['circle', 'triangle', 'rect', 'rectRot', 'star']; -export function ComparisonChart({ comparisonData, schools, metric, metricLabel }: ComparisonChartProps) { +export function ComparisonChart({ comparisonData, schools, metric, metricLabel, nationalByYear }: ComparisonChartProps) { const isMobile = useIsMobile(); const [focusedUrn, setFocusedUrn] = useState(null); @@ -54,34 +58,48 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel } return
No data available
; } - // Union of years across all schools — coverage differs between them. - const years = [ - ...new Set(schools.flatMap((s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => d.year) ?? [])), - ].sort((a, b) => a - b); + // Pure, tested series construction: union of years with cancelled / + // unpublished years kept as real gaps, plus the England overlay. + const built = buildCompareChart(comparisonData, schools, metric, nationalByYear); + const { years } = built; - const datasets: ChartDataset<'line'>[] = schools.map((school, index) => { - const data = comparisonData[String(school.urn)]; - const color = CHART_COLORS[index % CHART_COLORS.length]; + const datasets: ChartDataset<'line'>[] = built.schoolDatasets.map((series) => { + const school = schools[series.schoolIndex]; + const color = CHART_COLORS[series.schoolIndex % CHART_COLORS.length]; const dimmed = focusedUrn !== null && focusedUrn !== school.urn; return { - label: school.school_name, - data: years.map((year) => { - const yearData = data?.yearly_data.find((d) => d.year === year); - if (!yearData) return null; - return yearData[metric as keyof typeof yearData] as number | null; - }), + label: series.label, + data: series.data, borderColor: dimmed ? rgbToRgba(color, 0.2) : color, backgroundColor: dimmed ? 'transparent' : rgbToRgba(color, 0.1), borderWidth: focusedUrn === school.urn ? 3 : dimmed ? 1.5 : 2, - pointStyle: POINT_STYLES[index % POINT_STYLES.length], + pointStyle: POINT_STYLES[series.schoolIndex % POINT_STYLES.length], pointRadius: dimmed ? 2 : isMobile ? 3 : 4, pointHoverRadius: isMobile ? 5 : 6, tension: 0.3, - spanGaps: true, + // Never bridge missing years — gaps are information (COVID + // cancellations, unpublished 2021/22, schools that opened later). + spanGaps: false, }; }); + if (built.englandDataset) { + datasets.push({ + label: built.englandDataset.label, + data: built.englandDataset.data, + borderColor: 'rgba(109, 104, 95, 0.9)', + backgroundColor: 'transparent', + borderWidth: 1.5, + borderDash: built.englandDataset.borderDash, + pointStyle: 'line', + pointRadius: 0, + pointHoverRadius: 4, + tension: 0, + spanGaps: false, + }); + } + const chartData = { labels: years.map(formatAcademicYear), datasets, @@ -222,6 +240,12 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
+ {built.showUnpublished202122Note && ( +

+ No national tests were held in 2019/20 and 2020/21 (COVID), and DfE didn't publish + school-level figures for 2021/22 — the England average is shown for that year. +

+ )} ); } diff --git a/nextjs-app/components/ComparisonView.module.css b/nextjs-app/components/ComparisonView.module.css index b6f1af6..4b301e4 100644 --- a/nextjs-app/components/ComparisonView.module.css +++ b/nextjs-app/components/ComparisonView.module.css @@ -28,8 +28,15 @@ color: var(--text-secondary, #5c564d); margin: 0; line-height: 1.6; + max-width: 60ch; } +.headerActions { + display: flex; + gap: 0.75rem; + align-items: center; + flex-wrap: wrap; +} /* Phase Tabs */ .phaseTabs { @@ -72,408 +79,136 @@ background: var(--accent-coral-darker, #9c3f26); } -/* Metric Selector */ -.metricSelector { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; - padding: 1.5rem; - margin-bottom: 2rem; +/* Sticky school bar — column identity while scrolling; horizontal scroll on + narrow screens. Offset by the sticky site header's height (Navigation is + position: sticky, top: 0) so this bar pins just below it instead of + sliding underneath and being hidden. Header ≈ 65px desktop / 57px mobile. */ +.schoolBar { + position: sticky; + top: 65px; + z-index: 10; + background: var(--bg-primary, #faf7f2); display: flex; - align-items: center; - flex-wrap: wrap; - gap: 1rem; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); -} - -.metricLabel { - font-size: 0.9375rem; - font-weight: 600; - color: var(--text-primary, #1a1612); - white-space: nowrap; -} - -.metricSelect { - flex: 1; - max-width: 400px; - padding: 0.625rem 1rem; - font-size: 0.9375rem; - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 8px; - background: var(--bg-card, white); - color: var(--text-primary, #1a1612); - cursor: pointer; - transition: all 0.2s ease; -} - -.metricSelect:hover { - border-color: var(--accent-coral, #e07256); -} - -.metricSelect:focus { - outline: none; - border-color: var(--accent-coral, #e07256); - box-shadow: 0 0 0 3px var(--accent-coral-bg); -} - -.metricSelect optgroup { - font-weight: 700; - color: var(--text-primary, #1a1612); - background: var(--bg-secondary, #f3ede4); - padding: 0.5rem 0; -} - -.metricSelect option { - font-weight: 400; - color: var(--text-secondary, #5c564d); - padding: 0.375rem 1rem; -} - -/* Schools Section */ -.schoolsSection { - margin-bottom: 2rem; -} - -.schoolsGrid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); - gap: 1.5rem; -} - -.schoolCard { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-left: 3px solid var(--accent-teal, #2d7d7d); - border-radius: 12px; - padding: 1.5rem; - position: relative; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); - transition: all 0.3s ease; - display: flex; - flex-direction: column; -} - -.schoolCard:hover { - box-shadow: var(--shadow-medium, 0 4px 20px rgba(26, 22, 18, 0.1)); - transform: translateY(-2px); -} - -.removeButton { - position: absolute; - top: 0.75rem; - right: 0.75rem; - width: 28px; - height: 28px; - display: flex; - align-items: center; - justify-content: center; - background: var(--accent-coral, #e07256); - color: white; - border: none; - border-radius: 50%; - font-size: 1.25rem; - line-height: 1; - cursor: pointer; - transition: all 0.2s ease; -} - -.removeButton:hover { - background: var(--accent-coral-dark, #c45a3f); - transform: scale(1.1); -} - -.schoolName { - font-size: 1.125rem; - font-weight: 600; - margin-bottom: 0.75rem; - padding-right: 2rem; - line-height: 1.3; - font-family: var(--font-playfair), 'Playfair Display', serif; -} - -.schoolName a { - color: var(--text-primary, #1a1612); - text-decoration: none; - transition: color 0.2s ease; -} - -.schoolName a:hover { - color: var(--accent-coral-dark, #b04a2e); -} - -.schoolMeta { - display: flex; - flex-direction: column; - gap: 0.5rem; - margin-bottom: 1rem; - flex: 1; -} - -.metaItem { - font-size: 0.875rem; - color: var(--text-secondary, #5c564d); - display: flex; - align-items: center; - gap: 0.25rem; -} - -.latestValue { - margin-top: auto; - padding-top: 1rem; - border-top: 1px solid var(--border-color, #e5dfd5); - text-align: center; - background: var(--bg-secondary, #f3ede4); - margin-left: -1.5rem; - margin-right: -1.5rem; - margin-bottom: -1.5rem; - padding: 1.25rem 1.5rem; - border-radius: 0 0 12px 9px; -} - -.latestLabel { - font-size: 0.75rem; - color: var(--text-muted, #8a847a); - margin-bottom: 0.25rem; - text-transform: uppercase; - letter-spacing: 0.05em; -} - -.latestNumber { - font-size: 1.75rem; - font-weight: 700; - color: var(--accent-teal, #2d7d7d); -} - -/* Chart Section */ -.chartSection { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; - padding: 2rem; - margin-bottom: 2rem; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); -} - -.sectionTitle { - font-size: 1.5rem; - font-weight: 600; - color: var(--text-primary, #1a1612); - margin-bottom: 1.5rem; - padding-bottom: 0.75rem; - border-bottom: 2px solid var(--border-color, #e5dfd5); - font-family: var(--font-playfair), 'Playfair Display', serif; - display: flex; - align-items: center; - gap: 0.5rem; -} - -.sectionTitle::before { - content: ''; - display: inline-block; - width: 4px; - height: 1em; - background: var(--accent-coral, #e07256); - border-radius: 2px; -} - -.chartContainer { - width: 100%; - height: 400px; - position: relative; -} - -.loadingMessage { - text-align: center; - padding: 3rem; - color: var(--text-secondary, #5c564d); - font-size: 1rem; -} - -/* Table Section */ -.tableSection { - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; - padding: 2rem; - margin-bottom: 2rem; - box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); -} - -.tableWrapper { + gap: 0.75rem; overflow-x: auto; - max-width: 100%; - margin-top: 1rem; + padding: 0.75rem 0; + border-bottom: 1px solid var(--border-light, #e5dfd5); -webkit-overflow-scrolling: touch; } -/* Right-edge fade so phone users see the comparison table scrolls. - Otherwise the wider-than-viewport table silently clips. */ -@media (max-width: 640px) { - .tableWrapper { - -webkit-mask-image: linear-gradient(to right, #000 calc(100% - 28px), transparent); - mask-image: linear-gradient(to right, #000 calc(100% - 28px), transparent); - } +.schoolChip { + flex: 1 1 0; + min-width: 180px; + background: var(--bg-card, white); + border: 1px solid var(--border-light, #e5dfd5); + border-top: 3px solid var(--accent-coral, #e07256); + border-radius: 8px; + box-shadow: var(--shadow-soft, 0 2px 8px rgba(26, 22, 18, 0.06)); + padding: 0.55rem 0.75rem; + display: flex; + gap: 0.55rem; + align-items: center; } -.comparisonTable { - width: 100%; - border-collapse: separate; - border-spacing: 0; - font-size: 0.9375rem; +.chipDot { + width: 11px; + height: 11px; + border-radius: 50%; + flex: none; } -.comparisonTable thead { - background: var(--bg-secondary, #f3ede4); +.chipText { + min-width: 0; } -.comparisonTable th { - padding: 1rem; - text-align: left; +.chipName { + display: block; font-weight: 600; + font-size: 0.92rem; + line-height: 1.25; color: var(--text-primary, #1a1612); - border-bottom: 2px solid var(--border-color, #e5dfd5); - background: var(--bg-secondary, #f3ede4); + text-decoration: none; +} + +.chipName:hover { + color: var(--accent-coral-dark, #b04a2e); +} + +/* Full name on desktop, short name on the compact mobile pills. */ +.chipNameShort { + display: none; +} + +.chipMeta { + display: block; + font-size: 0.78rem; + color: var(--text-muted, #6d685f); white-space: nowrap; - text-transform: uppercase; - font-size: 0.75rem; - letter-spacing: 0.05em; + overflow: hidden; + text-overflow: ellipsis; } -.comparisonTable td { - padding: 1rem; - border-bottom: 1px solid var(--border-color, #e5dfd5); - color: var(--text-secondary, #5c564d); - text-align: left; - background: var(--bg-card, white); -} - -/* Sticky first column (Year) so labels remain visible while scrolling */ -.comparisonTable th:first-child, -.comparisonTable td:first-child { - position: sticky; - left: 0; - z-index: 1; - box-shadow: 2px 0 4px -2px rgba(26, 22, 18, 0.08); -} - -.comparisonTable thead th:first-child { - z-index: 2; -} - -.comparisonTable tbody tr:hover td:first-child { +.chipRemove { + margin-left: auto; + border: none; background: var(--bg-secondary, #f3ede4); + color: var(--text-muted, #6d685f); + border-radius: 50%; + width: 22px; + height: 22px; + cursor: pointer; + flex: none; + font-size: 0.9rem; + line-height: 1; } -.comparisonTable tbody tr:last-child td { - border-bottom: none; +.footnote { + font-size: 0.78rem; + color: var(--text-muted, #6d685f); + margin-top: 2.5rem; + border-top: 1px solid var(--border-light, #e5dfd5); + padding-top: 1rem; + max-width: 75ch; } -.comparisonTable tbody tr:hover { - background: var(--bg-secondary, #f3ede4); -} - -.yearCell { - font-weight: 700; - color: var(--accent-gold, #c9a227); -} - -/* Empty State */ -.emptyState { - text-align: center; - padding: 4rem 2rem; - background: var(--bg-card, white); - border: 1px solid var(--border-color, #e5dfd5); - border-radius: 12px; -} - -.emptyStateTitle { - font-size: 1.5rem; - font-weight: 600; - color: var(--text-primary, #1a1612); - margin-bottom: 0.5rem; - font-family: var(--font-playfair), 'Playfair Display', serif; -} - -.emptyStateDescription { - font-size: 1rem; - color: var(--text-secondary, #5c564d); - max-width: 400px; - margin: 0 auto 1.5rem; -} - -.metricDescription { - margin-top: 0.5rem; - font-size: 0.85rem; - color: var(--text-secondary); - max-width: 600px; - flex-basis: 100%; - margin-top: 0.25rem; -} - -.progressNote { - background: var(--bg-secondary); - border-left: 3px solid var(--accent-teal); - padding: 0.75rem 1rem; - margin: 0 0 1.5rem; - font-size: 0.875rem; - color: var(--text-secondary); - border-radius: 0 var(--radius-sm) var(--radius-sm) 0; -} - - -/* Responsive Design */ -@media (max-width: 768px) { - .headerContent { - flex-direction: column; - align-items: stretch; +/* Mobile: the sticky school bar becomes compact, horizontally-scrollable + pills with short names (matching the mobile mockup) instead of full-width + cards whose names wrap to several lines. */ +@media (max-width: 640px) { + /* The mobile Navigation header is shorter (≈57px). */ + .schoolBar { + top: 57px; } - .header h1 { - font-size: 1.75rem; + .schoolChip { + flex: 0 0 auto; + min-width: 0; + border-top-width: 2px; + border-radius: 999px; + padding: 0.35rem 0.7rem; + box-shadow: none; } - .metricSelector { - flex-direction: column; - align-items: stretch; - padding: 1rem; - border-radius: 8px; + .chipName { + font-size: 0.85rem; + white-space: nowrap; } - .metricSelect { - max-width: 100%; + .chipNameFull { + display: none; } - .schoolsGrid { - grid-template-columns: 1fr; + .chipNameShort { + display: inline; } - .chartSection, - .tableSection { - padding: 1rem; - border-radius: 8px; + .chipMeta { + display: none; } - .chartContainer { - /* Taller than desktop's proportion would suggest: the chip legend row - sits inside, and the in-chart title/legend/axis titles are gone, so - nearly all of this is plot area. */ - height: 340px; - } - - .comparisonTable { - font-size: 0.875rem; - } - - .comparisonTable th, - .comparisonTable td { - padding: 0.75rem 0.5rem; - } - - .latestValue { - margin-left: -1rem; - margin-right: -1rem; - margin-bottom: -1rem; - padding: 1rem; - border-radius: 0 0 8px 5px; + .chipRemove { + width: 18px; + height: 18px; + font-size: 0.75rem; } } diff --git a/nextjs-app/components/ComparisonView.tsx b/nextjs-app/components/ComparisonView.tsx index e1f929d..600794e 100644 --- a/nextjs-app/components/ComparisonView.tsx +++ b/nextjs-app/components/ComparisonView.tsx @@ -1,49 +1,42 @@ /** - * ComparisonView Component - * Client-side comparison interface with phase tabs, charts, and tables + * ComparisonView — the parent-first compare screen: a sticky school bar and + * six sections (At a glance / Ofsted / Academics / Getting a place / Who + * goes there / Explore trends), every number anchored against the England + * average or the computed state-school benchmark with provenance-correct + * labels. Layout and copy follow the reviewed mockups + * (docs/superpowers/specs/mockups/). */ 'use client'; import { useEffect, useRef, useState } from 'react'; import { useRouter, usePathname, useSearchParams } from 'next/navigation'; -import dynamic from 'next/dynamic'; import { useComparison } from '@/hooks/useComparison'; -const ComparisonChart = dynamic( - () => import('./ComparisonChart').then((m) => m.ComparisonChart), - { ssr: false }, -); import { SchoolSearchModal } from './SchoolSearchModal'; import { EmptyState } from './EmptyState'; -import { LoadingSkeleton } from './LoadingSkeleton'; -import type { ComparisonData, MetricDefinition, School } from '@/lib/types'; -import { formatPercentage, formatProgress, formatAcademicYear, CHART_COLORS, CHART_TEXT_COLORS, schoolUrl } from '@/lib/utils'; +import { CompareAtAGlance } from './compare/CompareAtAGlance'; +import { CompareOfsted } from './compare/CompareOfsted'; +import { CompareAcademics } from './compare/CompareAcademics'; +import { CompareAdmissions } from './compare/CompareAdmissions'; +import { CompareCommunity } from './compare/CompareCommunity'; +import { TrendsExplorer, PRIMARY_CATEGORIES, SECONDARY_CATEGORIES } from './compare/TrendsExplorer'; +import type { + Benchmarks, + ComparisonData, + MetricDefinition, + NationalAverages, + School, +} from '@/lib/types'; +import { CHART_COLORS, schoolUrl, shortName } from '@/lib/utils'; import { fetchComparison } from '@/lib/api'; import { track } from '@/lib/analytics'; import styles from './ComparisonView.module.css'; -const PRIMARY_CATEGORIES = ['expected', 'higher', 'progress', 'average', 'gender', 'equity', 'context', 'absence', 'trends']; -const SECONDARY_CATEGORIES = ['gcse']; - -const PRIMARY_OPTGROUPS: { label: string; category: string }[] = [ - { label: 'Expected Standard', category: 'expected' }, - { label: 'Higher Standard', category: 'higher' }, - { label: 'Progress Scores', category: 'progress' }, - { label: 'Average Scores', category: 'average' }, - { label: 'Gender Performance', category: 'gender' }, - { label: 'Equity (Disadvantaged)', category: 'equity' }, - { label: 'School Context', category: 'context' }, - { label: 'Absence', category: 'absence' }, - { label: '3-Year Trends', category: 'trends' }, -]; - -const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [ - { label: 'GCSE Performance', category: 'gcse' }, -]; - interface ComparisonViewProps { initialData: Record | null; + initialNationalAverages?: NationalAverages; + initialBenchmarks?: Benchmarks; initialUrns: number[]; metrics: MetricDefinition[]; selectedMetric: string; @@ -51,6 +44,8 @@ interface ComparisonViewProps { export function ComparisonView({ initialData, + initialNationalAverages, + initialBenchmarks, initialUrns, metrics, selectedMetric: initialMetric, @@ -58,11 +53,15 @@ export function ComparisonView({ const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - const { selectedSchools, removeSchool, addSchool, replaceSchools, isInitialized } = useComparison(); + const { selectedSchools, removeSchool, replaceSchools, isInitialized } = useComparison(); const [selectedMetric, setSelectedMetric] = useState(initialMetric); const [isModalOpen, setIsModalOpen] = useState(false); const [comparisonData, setComparisonData] = useState(initialData); + const [nationalAverages, setNationalAverages] = useState( + initialNationalAverages, + ); + const [benchmarks, setBenchmarks] = useState(initialBenchmarks); const [shareConfirm, setShareConfirm] = useState(false); const [comparePhase, setComparePhase] = useState<'primary' | 'secondary'>('primary'); // Tracks whether the user has explicitly clicked a phase tab. @@ -77,24 +76,27 @@ export function ComparisonView({ if (!isInitialized) return; if (initialUrns.length > 0 && initialData) { const urlSchools = initialUrns - .map(urn => initialData[String(urn)]?.school_info) + .map((urn) => initialData[String(urn)]?.school_info) .filter((info): info is NonNullable => Boolean(info)); const sameSet = urlSchools.length === selectedSchools.length && - urlSchools.every(s => selectedSchools.some(sel => sel.urn === s.urn)); + urlSchools.every((s) => selectedSchools.some((sel) => sel.urn === s.urn)); if (urlSchools.length > 0 && !sameSet) { replaceSchools(urlSchools); } } }, [isInitialized]); // eslint-disable-line react-hooks/exhaustive-deps - // Sync URL with selected schools + const urnKey = selectedSchools.map((s) => s.urn).join(','); + + // Sync the URL with the selection + metric. Pure navigation state — no + // fetching here: metric changes are presentational (the data is already + // client-side) and must not refire the comparison request. useEffect(() => { - const urns = selectedSchools.map((s) => s.urn).join(','); const params = new URLSearchParams(searchParams); - if (urns) { - params.set('urns', urns); + if (urnKey) { + params.set('urns', urnKey); } else { params.delete('urns'); } @@ -103,52 +105,70 @@ export function ComparisonView({ const newUrl = `${pathname}?${params.toString()}`; router.replace(newUrl, { scroll: false }); + }, [urnKey, selectedMetric, pathname, searchParams, router]); - // Fetch comparison data - if (selectedSchools.length > 0) { - fetchComparison(urns, { cache: 'no-store' }) - .then((data) => { - setComparisonData(data.comparison); - }) - .catch((err) => { - // Keep whatever we already have (SSR data or a previous fetch) rather - // than blanking the chart — a transient refetch failure shouldn't - // destroy a working comparison the user is looking at. - console.error('Failed to fetch comparison:', err); - }); - } else { - setComparisonData(null); - } - }, [selectedSchools, selectedMetric, pathname, searchParams, router]); + // Fetch when the school set changes, but only for schools we don't already + // have data for. This skips the refetch of SSR-rendered data on load AND + // avoids a network call when a school is merely removed. A ref holds the + // latest data so the effect can read it without re-running on every fetch. + // + // Correctness note: we must NOT null the data on a transient empty urnKey. + // On mount the basket is empty for a beat before it hydrates from the URL, + // and blanking here (then skipping the refetch because SSR "covers" the set) + // was leaving the page empty on refresh. The render already shows the empty + // state whenever `selectedSchools` is empty, so stale data for deselected + // schools is harmless — it's simply unused. + const comparisonDataRef = useRef(comparisonData); + comparisonDataRef.current = comparisonData; - // Classify schools by phase using comparison data - const classifySchool = (school: School): 'primary' | 'secondary' => { + useEffect(() => { + if (!isInitialized || !urnKey) return; + + const have = comparisonDataRef.current ?? {}; + const covered = urnKey.split(',').every((urn) => have[urn] != null); + if (covered) return; + + fetchComparison(urnKey, { cache: 'no-store' }) + .then((data) => { + setComparisonData(data.comparison); + setNationalAverages(data.national_averages); + setBenchmarks(data.benchmarks); + }) + .catch((err) => { + // Keep whatever we already have (SSR data or a previous fetch) rather + // than blanking the page — a transient refetch failure shouldn't + // destroy a working comparison the user is looking at. + console.error('Failed to fetch comparison:', err); + }); + }, [urnKey, isInitialized]); + + const primarySchools = selectedSchools.filter((school) => { const info = comparisonData?.[school.urn]?.school_info; - if (info?.attainment_8_score != null) return 'secondary'; - if (info?.rwm_expected_pct != null) return 'primary'; - // Fallback: check yearly data - const yearlyData = comparisonData?.[school.urn]?.yearly_data; - if (yearlyData?.some((d: any) => d.attainment_8_score != null)) return 'secondary'; - return 'primary'; - }; + const hasPrimaryData = + info?.rwm_expected_pct != null || + comparisonData?.[school.urn]?.yearly_data?.some((d) => d.rwm_expected_pct != null); + if (hasPrimaryData) return true; + return school.phase?.toLowerCase().includes('primary') || false; + }); - const primarySchools = selectedSchools.filter(s => classifySchool(s) === 'primary'); - const secondarySchools = selectedSchools.filter(s => classifySchool(s) === 'secondary'); + const secondarySchools = selectedSchools.filter((school) => { + const info = comparisonData?.[school.urn]?.school_info; + const hasSecondaryData = + info?.attainment_8_score != null || + comparisonData?.[school.urn]?.yearly_data?.some((d) => d.attainment_8_score != null); + if (hasSecondaryData) return true; + return school.phase?.toLowerCase().includes('secondary') || false; + }); - // Auto-select tab with more schools and sync the metric to match the detected phase. - // This fixes the case where the URL carries a primary metric (e.g. rwm_expected_pct) - // but the shortlisted schools are secondary — the phase tab switches but the metric - // needs to follow, otherwise all secondary cards show "–" for a primary-only field. + // Auto-select tab with more schools and sync the metric to match the phase. useEffect(() => { if (!comparisonData || selectedSchools.length === 0) return; if (phaseLockedByUser.current) return; const newPhase = secondarySchools.length > primarySchools.length ? 'secondary' : 'primary'; setComparePhase(newPhase); - // Only reset the metric when it doesn't belong to the newly detected phase. - // This preserves a correct metric that came from the URL (e.g. metric=attainment_8_score). const phaseCategories = newPhase === 'secondary' ? SECONDARY_CATEGORIES : PRIMARY_CATEGORIES; const metricFitsPhase = metrics.some( - (m) => m.key === selectedMetric && phaseCategories.includes(m.category) + (m) => m.key === selectedMetric && phaseCategories.includes(m.category), ); if (!metricFitsPhase) { setSelectedMetric(newPhase === 'secondary' ? 'attainment_8_score' : 'rwm_expected_pct'); @@ -158,29 +178,24 @@ export function ComparisonView({ const handlePhaseChange = (phase: 'primary' | 'secondary') => { phaseLockedByUser.current = true; setComparePhase(phase); - const defaultMetric = phase === 'secondary' ? 'attainment_8_score' : 'rwm_expected_pct'; - setSelectedMetric(defaultMetric); + setSelectedMetric(phase === 'secondary' ? 'attainment_8_score' : 'rwm_expected_pct'); }; // compare_viewed: fire once after the page has its first selection. - // We watch `selectedSchools.length` going from 0 → ≥1 so the event is - // sent only when there's actual content to view, not for empty arrivals. const compareViewedRef = useRef(false); useEffect(() => { if (compareViewedRef.current) return; if (selectedSchools.length === 0) return; compareViewedRef.current = true; - const primaryCount = selectedSchools.filter(s => s.phase?.toLowerCase().includes('primary')).length; + const primaryCount = selectedSchools.filter((s) => + s.phase?.toLowerCase().includes('primary'), + ).length; const secondaryCount = selectedSchools.length - primaryCount; - const phaseMix = primaryCount === 0 ? 'all_secondary' : secondaryCount === 0 ? 'all_primary' : 'mixed'; + const phaseMix = + primaryCount === 0 ? 'all_secondary' : secondaryCount === 0 ? 'all_primary' : 'mixed'; track('compare_viewed', { school_count: selectedSchools.length, phase_mix: phaseMix }); }, [selectedSchools]); - const handleMetricChange = (metric: string) => { - track('compare_metric_changed', { metric, phase: comparePhase }); - setSelectedMetric(metric); - }; - const handleRemoveSchool = (urn: number) => { removeSchool(urn); track('compare_school_removed', { urn, from: 'compare' }); @@ -191,21 +206,22 @@ export function ComparisonView({ const count = selectedSchools.length; const shareData = { title: 'School comparison · SchoolCompare', - text: count > 0 - ? `Comparing ${count} school${count === 1 ? '' : 's'} on SchoolCompare` - : 'SchoolCompare', + text: + count > 0 + ? `Comparing ${count} school${count === 1 ? '' : 's'} on SchoolCompare` + : 'SchoolCompare', url, }; - // Prefer the native share sheet on platforms that support it (iOS / Android). - // canShare is feature-detected because Safari iOS exposes share() but - // some configurations refuse the payload. - if (typeof navigator !== 'undefined' && navigator.share && (!navigator.canShare || navigator.canShare(shareData))) { + if ( + typeof navigator !== 'undefined' && + navigator.share && + (!navigator.canShare || navigator.canShare(shareData)) + ) { try { await navigator.share(shareData); track('compare_shared', { method: 'native', school_count: count }); return; } catch (err) { - // User cancelled — bail silently. Any other error falls through to clipboard. if ((err as DOMException)?.name === 'AbortError') return; } } @@ -214,27 +230,22 @@ export function ComparisonView({ track('compare_shared', { method: 'clipboard', school_count: count }); setShareConfirm(true); setTimeout(() => setShareConfirm(false), 2000); - } catch { /* fallback: do nothing */ } + } catch { + /* fallback: do nothing */ + } }; const isPrimary = comparePhase === 'primary'; - const allowedCategories = isPrimary ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES; - const optgroups = isPrimary ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS; - const filteredMetrics = metrics.filter(m => allowedCategories.includes(m.category)); const activeSchools = isPrimary ? primarySchools : secondarySchools; - // Get metric definition - const currentMetricDef = metrics.find((m) => m.key === selectedMetric); - const metricLabel = currentMetricDef?.label || selectedMetric; - - // No schools selected if (selectedSchools.length === 0) { return (

Compare Schools

- Add schools to your comparison basket to see side-by-side performance data + Add schools to your comparison basket to see them side by side — inspection results, + academics, admissions and community.

@@ -252,39 +263,46 @@ export function ComparisonView({ ); } - // Build filtered comparison data for active phase + // Build filtered comparison data for the active phase const activeComparisonData: Record = {}; if (comparisonData) { - activeSchools.forEach(s => { + activeSchools.forEach((s) => { if (comparisonData[s.urn]) { activeComparisonData[s.urn] = comparisonData[s.urn]; } }); } - - // Get years for table - const years = - Object.keys(activeComparisonData).length > 0 - ? activeComparisonData[Object.keys(activeComparisonData)[0]].yearly_data.map((d) => d.year) - : []; + const hasData = Object.keys(activeComparisonData).length > 0; return (
- {/* Header */}

Compare Schools

- Comparing {selectedSchools.length} school{selectedSchools.length !== 1 ? 's' : ''} + {selectedSchools.length} school{selectedSchools.length !== 1 ? 's' : ''} side by side + — each number anchored against the England average so you can tell at a glance + what's typical and what stands out.

-
+
@@ -292,20 +310,22 @@ export function ComparisonView({
{/* Phase Tabs */} -
- - -
+ {secondarySchools.length > 0 && primarySchools.length > 0 && ( +
+ + +
+ )} {activeSchools.length === 0 ? ( ) : ( <> - {/* Metric Selector */} -
- - - {currentMetricDef?.description && ( -

{currentMetricDef.description}

- )} -
- - {/* Progress score explanation */} - {selectedMetric.includes('progress') && ( -

- Progress scores measure pupils' progress from KS1 to KS2. A score of 0 equals the national average; positive scores are above average. -

- )} - - {/* School Cards */} -
-
- {activeSchools.map((school, index) => ( -
- -

- {school.school_name} -

-
- {school.local_authority && ( - {school.local_authority} - )} - {school.school_type && ( - {school.school_type} - )} -
- - {/* Latest metric value */} - {activeComparisonData[school.urn] && ( -
-
{metricLabel}
- {/* Text uses the AA-dark variant; the swatch dot keeps the true series colour */} -
- - {(() => { - const yearlyData = activeComparisonData[school.urn].yearly_data; - if (yearlyData.length === 0) return '-'; - - const latestData = yearlyData[yearlyData.length - 1]; - const value = latestData[selectedMetric as keyof typeof latestData]; - - if (value === null || value === undefined) return '-'; - - if (selectedMetric.includes('progress')) { - return formatProgress(value as number); - } else if (selectedMetric.includes('pct') || selectedMetric.includes('rate')) { - return formatPercentage(value as number); - } else { - return typeof value === 'number' ? value.toFixed(1) : String(value); - } - })()} -
-
- )} -
- ))} -
-
- - {/* Comparison Chart */} - {Object.keys(activeComparisonData).length > 0 ? ( -
-

Performance Over Time

-
- + {activeSchools.map((school, index) => ( +
+
-
- ) : activeSchools.length > 0 ? ( -
- -
- ) : null} + ))} +
- {/* Comparison Table */} - {Object.keys(activeComparisonData).length > 0 && years.length > 0 && ( -
-

Detailed Comparison

-
- - - - - {activeSchools.map((school) => ( - - ))} - - - - {years.map((year) => ( - - - {activeSchools.map((school) => { - const schoolData = activeComparisonData[school.urn]; - if (!schoolData) return ; + {hasData && ( + <> + + + + + + - const yearData = schoolData.yearly_data.find((d) => d.year === year); - if (!yearData) return ; - - const value = yearData[selectedMetric as keyof typeof yearData]; - - if (value === null || value === undefined) { - return ; - } - - let displayValue: string; - if (selectedMetric.includes('progress')) { - displayValue = formatProgress(value as number); - } else if (selectedMetric.includes('pct') || selectedMetric.includes('rate')) { - displayValue = formatPercentage(value as number); - } else { - displayValue = typeof value === 'number' ? value.toFixed(1) : String(value); - } - - return ; - })} - - ))} - -
Year{school.school_name}
{formatAcademicYear(year)}---{displayValue}
-
-
+

+ Sources: DfE Compare School Performance (KS2/KS4 results), Ofsted inspection + outcomes, DfE school admissions data, school census. England averages for test + results are official DfE figures; other benchmarks are state-school averages + computed from our dataset. Following DfE practice, figures based on 5 or fewer + pupils are suppressed and shown as "no data". +

+ )} )} - {/* School Search Modal */} setIsModalOpen(false)} />
); diff --git a/nextjs-app/components/DotStrip.module.css b/nextjs-app/components/DotStrip.module.css new file mode 100644 index 0000000..2b7f380 --- /dev/null +++ b/nextjs-app/components/DotStrip.module.css @@ -0,0 +1,102 @@ +.row { + margin: 1.1rem 0 1.6rem; +} + +.head { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 1rem; + flex-wrap: wrap; +} + +.title { + font-weight: 600; + font-size: 0.95rem; +} + +.headNote { + font-size: 0.8rem; + color: var(--text-muted); +} + +.strip { + position: relative; + height: 34px; + margin-top: 0.45rem; +} + +.track { + position: absolute; + left: 0; + right: 0; + top: 15px; + height: 4px; + border-radius: 2px; + background: var(--bg-secondary); +} + +.anchorTick { + position: absolute; + top: 4px; + width: 2px; + height: 26px; + background: var(--text-muted); +} + +.anchorLabel { + position: absolute; + top: -14px; + transform: translateX(-50%); + font-size: 0.7rem; + color: var(--text-muted); + white-space: nowrap; +} + +.point { + position: absolute; + top: 9px; + width: 16px; + height: 16px; + border-radius: 50%; + transform: translateX(-50%); + border: 2px solid var(--bg-card); + box-shadow: 0 0 0 1px rgba(26, 22, 18, 0.08); +} + +.pointLabel { + position: absolute; + top: 27px; + transform: translateX(-50%); + font-size: 0.72rem; + font-weight: 600; + font-variant-numeric: tabular-nums; + color: var(--text-secondary); +} + +.pointLabelAbove { + top: -6px; +} + +@media (max-width: 760px) { + .row { + margin: 0.9rem 0 1.3rem; + } + + .title { + font-size: 0.82rem; + } + + .strip { + height: 32px; + } + + .point { + width: 14px; + height: 14px; + } + + .pointLabel { + font-size: 0.64rem; + } +} diff --git a/nextjs-app/components/DotStrip.tsx b/nextjs-app/components/DotStrip.tsx new file mode 100644 index 0000000..cd3b843 --- /dev/null +++ b/nextjs-app/components/DotStrip.tsx @@ -0,0 +1,97 @@ +/** + * DotStrip — the compare screen's signature element: one measure per strip, + * every school's dot on a shared track, anchored by a grey England-average + * tick so "right of the tick = above average" needs no domain knowledge. + */ + +'use client'; + +import { stripPositions } from '@/lib/compareLogic'; +import { CHART_COLORS, CHART_TEXT_COLORS } from '@/lib/utils'; +import styles from './DotStrip.module.css'; + +export interface DotStripProps { + label: string; + /** One value per school; index = the school's chart-colour index. */ + values: Array; + schoolNames: string[]; + /** Anchor tick, e.g. { value: 62, label: 'England 62%' }. Omit when the + * benchmark isn't available — the caller should say why in `headNote`. */ + anchor?: { value: number; label: string } | null; + min?: number; + max?: number; + unit?: string; + /** Tooltip on the measure label (plain-English definition). */ + tip?: string; + /** Small note on the right of the header row (e.g. the tick legend). */ + headNote?: string; +} + +export function DotStrip({ + label, + values, + schoolNames, + anchor = null, + min = 0, + max = 100, + unit = '%', + tip, + headNote, +}: DotStripProps) { + const points = stripPositions(values, min, max); + const span = max - min; + const anchorPos = + anchor != null + ? Math.min(100, Math.max(0, ((anchor.value - min) / span) * 100)) + : null; + + const ariaParts = [ + anchor ? `${anchor.label}` : null, + ...points.map( + (p) => `${schoolNames[p.schoolIndex] ?? `School ${p.schoolIndex + 1}`} ${p.value}${unit}`, + ), + ].filter(Boolean); + + return ( +
+
+ + {label} + + {headNote && {headNote}} +
+
+
+ {anchorPos != null && anchor && ( + <> + + + {anchor.label} + + + )} + {points.map((p) => ( + + + + {p.value} + + + ))} +
+
+ ); +} diff --git a/nextjs-app/components/compare/CompareAcademics.module.css b/nextjs-app/components/compare/CompareAcademics.module.css new file mode 100644 index 0000000..42b36ef --- /dev/null +++ b/nextjs-app/components/compare/CompareAcademics.module.css @@ -0,0 +1,18 @@ +.moreMeasures { + margin-top: 0.5rem; + border-top: 1px solid var(--border-light); + padding-top: 0.75rem; +} + +.moreMeasures summary { + cursor: pointer; + font-weight: 600; + font-size: 0.88rem; + color: var(--accent-coral-dark); +} + +.stripNote { + font-size: 0.78rem; + color: var(--text-muted); + margin: 0.5rem 0 0; +} diff --git a/nextjs-app/components/compare/CompareAcademics.tsx b/nextjs-app/components/compare/CompareAcademics.tsx new file mode 100644 index 0000000..5dfb78f --- /dev/null +++ b/nextjs-app/components/compare/CompareAcademics.tsx @@ -0,0 +1,294 @@ +/** + * How children do academically — tier-1 dot strips anchored on official + * England averages, tier-2 "More measures" one tap away, equity row against + * the computed state-school benchmark. Copy verbatim from the reviewed + * mockups; teacher-assessed measures are labelled as such. + */ + +'use client'; + +import { latestValues, verdict } from '@/lib/compareLogic'; +import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types'; +import { DotStrip } from '@/components/DotStrip'; +import { Cell, Chip, RowLabel, Section, SectionGrid, sectionStyles as s } from './sectionShared'; +import styles from './CompareAcademics.module.css'; + +interface StripSpec { + label: string; + metric: string; + anchorKey?: string; + tip?: string; + min?: number; + max?: number; + unit?: string; +} + +const TIER1_PRIMARY: StripSpec[] = [ + { + label: 'Reading, writing & maths — expected standard', + metric: 'rwm_expected_pct', + anchorKey: 'rwm_expected_pct', + tip: '% of Year 6 pupils reaching the expected standard in reading, writing and maths.', + }, + { label: 'Reading', metric: 'reading_expected_pct', anchorKey: 'reading_expected_pct' }, + { + label: 'Writing (teacher-assessed)', + metric: 'writing_expected_pct', + anchorKey: 'writing_expected_pct', + tip: 'Writing is assessed by teachers, not tested.', + }, + { label: 'Maths', metric: 'maths_expected_pct', anchorKey: 'maths_expected_pct' }, + { + label: 'Working at a higher standard than expected', + metric: 'rwm_high_pct', + anchorKey: 'rwm_high_pct', + tip: 'A high score in the reading and maths tests plus “greater depth” in teacher-assessed writing.', + }, +]; + +const TIER2_PRIMARY: StripSpec[] = [ + { + label: 'Grammar, punctuation & spelling — expected standard', + metric: 'gps_expected_pct', + anchorKey: 'gps_expected_pct', + }, + { + label: 'Science — expected standard (teacher-assessed)', + metric: 'science_expected_pct', + anchorKey: 'science_expected_pct', + tip: 'Teacher-assessed, like writing — there has been no KS2 science test since 2009, so comparisons are indicative.', + }, + { + label: 'Average scaled score — reading', + metric: 'reading_avg_score', + anchorKey: 'reading_avg_score', + min: 100, + max: 120, + unit: '', + }, + { + label: 'Average scaled score — maths', + metric: 'maths_avg_score', + anchorKey: 'maths_avg_score', + min: 100, + max: 120, + unit: '', + }, + { + label: 'Average scaled score — grammar, punctuation & spelling', + metric: 'gps_avg_score', + anchorKey: 'gps_avg_score', + min: 100, + max: 120, + unit: '', + }, +]; + +function Strip({ + spec, + data, + urns, + schoolNames, + national, +}: { + spec: StripSpec; + data: Record; + urns: number[]; + schoolNames: string[]; + national: Record | undefined; +}) { + const values = latestValues(data, urns, spec.metric).map((v) => + v != null ? Math.round(v) : null, + ); + const anchorValue = spec.anchorKey ? national?.[spec.anchorKey] : undefined; + const anchor = + anchorValue != null + ? { value: anchorValue, label: `England ${Math.round(anchorValue)}${spec.unit ?? '%'}` } + : null; + if (values.every((v) => v == null)) return null; + return ( + + ); +} + +export function CompareAcademics({ + schools, + data, + nationalAverages, + benchmarks, + isSecondary: propIsSecondary, +}: { + schools: School[]; + data: Record; + nationalAverages?: NationalAverages; + benchmarks?: Benchmarks; + isSecondary?: boolean; +}) { + const urns = schools.map((school) => school.urn); + const schoolNames = schools.map((school) => school.school_name); + const isSecondary = propIsSecondary !== undefined ? propIsSecondary : schools.some( + (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, + ); + + if (isSecondary) { + const att8 = latestValues(data, urns, 'attainment_8_score'); + const banding = urns.map((urn) => { + const rows = data[String(urn)]?.yearly_data ?? []; + for (let i = rows.length - 1; i >= 0; i--) { + if (rows[i].progress_8_banding) return rows[i].progress_8_banding as string; + } + return null; + }); + const grade5 = latestValues(data, urns, 'english_maths_strong_pass_pct'); + const ebacc = latestValues(data, urns, 'ebacc_entry_pct'); + const att8Anchor = nationalAverages?.secondary?.attainment_8_score; + + return ( +
+ + Attainment 8 + {schools.map((school, i) => ( + + {att8[i] != null ? ( + <> + {(att8[i] as number).toFixed(1)} + {att8Anchor != null && ( + England average {att8Anchor.toFixed(1)} + )} + + ) : ( + No data + )} + + ))} + + Progress 8 + {schools.map((school, i) => ( + + {banding[i] ? ( + + {banding[i]} + + ) : ( + No data + )} + + ))} + + + Grade 5+ in English & maths + + {schools.map((school, i) => ( + + {grade5[i] != null ? `${Math.round(grade5[i] as number)}%` : No data} + + ))} + + EBacc entry + {schools.map((school, i) => ( + + {ebacc[i] != null ? `${Math.round(ebacc[i] as number)}%` : No data} + + ))} + +
+ ); + } + + const national = nationalAverages?.primary; + const disadvantaged = latestValues(data, urns, 'rwm_expected_disadvantaged_pct'); + const disadvantagedAnchor = benchmarks?.primary?.disadvantaged_rwm_expected_pct ?? null; + + return ( +
+
+ {TIER1_PRIMARY.map((spec) => ( + + ))} + +
+ More measures — grammar, punctuation & spelling, science, average scaled scores + {TIER2_PRIMARY.map((spec) => ( + + ))} +

+ The scaled-score strips show the 100–120 window of the full 80–120 range; 100 is the + expected standard. Where an England tick is missing, the official figure isn't in + our dataset yet. +

+
+
+ + {disadvantaged.some((v) => v != null) && ( + + + Children from lower-income families + + {schools.map((school, i) => { + const value = disadvantaged[i]; + return ( + + {value != null ? ( + <> + + {Math.round(value)}% + {' '} + {disadvantagedAnchor != null && ( + + {verdict(value, disadvantagedAnchor, 5) === 'above' && + `Well above the ${Math.round(disadvantagedAnchor)}% state-school average`} + {verdict(value, disadvantagedAnchor, 5) === 'close' && + `Around the ${Math.round(disadvantagedAnchor)}% state-school average`} + {verdict(value, disadvantagedAnchor, 5) === 'below' && + `Below the ${Math.round(disadvantagedAnchor)}% state-school average`} + + )} + + ) : ( + No data + )} + + ); + })} + + )} +
+ ); +} diff --git a/nextjs-app/components/compare/CompareAdmissions.tsx b/nextjs-app/components/compare/CompareAdmissions.tsx new file mode 100644 index 0000000..9e0b4e4 --- /dev/null +++ b/nextjs-app/components/compare/CompareAdmissions.tsx @@ -0,0 +1,130 @@ +/** + * Getting a place — admissions framed the way the expert review requires: + * total applications are "named on N forms" (any preference rank, not + * head-to-head), one consistent chip metric (first-preference success), + * equal-preference and offers-vs-intake explanations up front. + */ + +'use client'; + +import { summariseAdmissions } from '@/lib/compareLogic'; +import type { ComparisonData, School } from '@/lib/types'; +import { CHART_COLORS } from '@/lib/utils'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +export function CompareAdmissions({ + schools, + data, +}: { + schools: School[]; + data: Record; +}) { + const rows = schools.map((school) => data[String(school.urn)]?.admissions ?? null); + const anyData = rows.some(Boolean); + const entryYear = rows.find(Boolean)?.year; + const entryLabel = entryYear + ? `September ${String(entryYear).slice(0, 4)} entry` + : 'the most recent admissions round'; + + if (!anyData) { + return ( +
+ <> +
+ ); + } + + return ( +
+ From the most recent admissions round ({entryLabel}). "First choice" means + families who ranked the school top of their application form — officially a "first + preference". Schools never see your ranking: places are decided only by the + school's admission criteria, so listing a school lower down never hurts your chances. + These are National Offer Day offers — waiting lists and appeals can change the final + intake. + + } + > + + + {schools.map((school, i) => { + const a = rows[i]; + return ( + + {a?.total_applications != null && a?.places_offered != null ? ( + <> + Named on {a.total_applications.toLocaleString('en-GB')} forms ·{' '} + {a.places_offered.toLocaleString('en-GB')} places + + ) : ( + No data + )} + + ); + })} + + + + + {schools.map((school, i) => { + const summary = summariseAdmissions(rows[i]); + return ( + + {summary.firstPrefPct != null ? ( + <> + {summary.firstPrefPct}%{' '} + {summary.chip && summary.chip.tone === 'warn' && ( + {summary.chip.text} + )} + + + + + ) : ( + No data + )} + + ); + })} + + + + + {schools.map((school, i) => { + const a = rows[i]; + const summary = summariseAdmissions(a); + let text: string | null = null; + if (summary.firstPrefPct != null) { + if (summary.firstPrefPct >= 100) { + text = `Every family who put ${school.school_name} first got a place.`; + } else if (summary.firstPrefPct >= 90) { + text = `Nearly every family who put ${school.school_name} first got a place.`; + } else if (a?.oversubscribed) { + text = + 'More first-choice applications than places — check the school’s admission criteria (for most non-faith primaries, distance decides).'; + } else { + text = `${summary.firstPrefPct}% of first-choice families received an offer.`; + } + } + return ( + + {text ? {text} : —} + + ); + })} + + +
+ ); +} diff --git a/nextjs-app/components/compare/CompareAtAGlance.tsx b/nextjs-app/components/compare/CompareAtAGlance.tsx new file mode 100644 index 0000000..f73f548 --- /dev/null +++ b/nextjs-app/components/compare/CompareAtAGlance.tsx @@ -0,0 +1,193 @@ +/** + * At a glance — the short version of every section below it. Copy verbatim + * from the reviewed mockups. Report-card cells summarise by counting graded + * areas (best first) and always NAME problem areas; safeguarding is a + * separate line, never a count. + */ + +'use client'; + +import { + latestValues, + ofstedDisplay, + summariseAdmissions, + verdict, + type ReportCardSummary, +} from '@/lib/compareLogic'; +import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +function ReportCardChips({ summary }: { summary: ReportCardSummary }) { + return ( + <> + Report card + + {summary.counts.map((c) => ( + + {c.count} area{c.count === 1 ? '' : 's'} {c.label} + + ))} + {summary.problems.map((p) => ( + + {p.areaLabel}: {p.label} + + ))} + + + {summary.allClear && 'No areas need attention · '} + {summary.safeguarding === 'met' && 'Safeguarding met'} + {summary.safeguarding === 'not_met' && 'Safeguarding not met'} + + + ); +} + +export function CompareAtAGlance({ + schools, + data, + nationalAverages, + benchmarks, + isSecondary: propIsSecondary, +}: { + schools: School[]; + data: Record; + nationalAverages?: NationalAverages; + benchmarks?: Benchmarks; + isSecondary?: boolean; +}) { + const urns = schools.map((school) => school.urn); + const isSecondary = propIsSecondary !== undefined ? propIsSecondary : schools.some( + (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, + ); + const headlineKey = isSecondary ? 'attainment_8_score' : 'rwm_expected_pct'; + const headlineValues = latestValues(data, urns, headlineKey); + const anchor = isSecondary + ? nationalAverages?.secondary?.attainment_8_score + : nationalAverages?.primary?.rwm_expected_pct; + const medianPupils = isSecondary + ? benchmarks?.secondary?.median_pupils + : benchmarks?.primary?.median_pupils; + + return ( +
+ + + {schools.map((school, i) => { + const display = ofstedDisplay(data[String(school.urn)]?.ofsted); + return ( + + {display.kind === 'report_card' && } + {(display.kind === 'graded' || display.kind === 'carried_forward') && ( + <> + + {display.gradeLabel} + + {display.carriedForward && Grade carried forward} + + )} + {display.kind === 'transitional' && ( + <> + + No overall grade + + Sub-judgements only + + )} + {display.kind === 'none' && No inspection in our dataset} + + ); + })} + + + + {schools.map((school, i) => { + const value = headlineValues[i]; + return ( + + {value != null ? ( + <> + {isSecondary ? value.toFixed(1) : `${Math.round(value)}%`}{' '} + {anchor != null && ( + + {verdict(value, anchor) === 'above' && 'Above England average'} + {verdict(value, anchor) === 'close' && 'Close to England average'} + {verdict(value, anchor) === 'below' && 'Below England average'} + + )} + {anchor != null && ( + + England average {isSecondary ? anchor.toFixed(1) : `${Math.round(anchor)}%`} + + )} + + ) : ( + No data + )} + + ); + })} + + + + {schools.map((school, i) => { + const summary = summariseAdmissions(data[String(school.urn)]?.admissions); + return ( + + {summary.chip ? ( + <> + {summary.chip.text} + {summary.interest && {summary.interest}} + + ) : ( + No admissions data + )} + + ); + })} + + + + {schools.map((school, i) => { + const census = data[String(school.urn)]?.census; + const pupils = census?.total_pupils ?? school.total_pupils ?? null; + let sizeNote: string | null = null; + if (pupils != null && medianPupils != null) { + if (pupils >= medianPupils * 1.5) sizeNote = 'Much larger than average'; + else if (pupils >= medianPupils * 1.1) sizeNote = 'Larger than average'; + else if (pupils <= medianPupils * 0.66) sizeNote = 'Much smaller than average'; + else if (pupils <= medianPupils * 0.9) sizeNote = 'Smaller than average'; + else sizeNote = 'About average size'; + } + return ( + + {pupils != null ? ( + <> + {pupils.toLocaleString('en-GB')} pupils + {sizeNote && {sizeNote}} + + ) : ( + No data + )} + + ); + })} + + +
+ ); +} diff --git a/nextjs-app/components/compare/CompareCommunity.tsx b/nextjs-app/components/compare/CompareCommunity.tsx new file mode 100644 index 0000000..9e14288 --- /dev/null +++ b/nextjs-app/components/compare/CompareCommunity.tsx @@ -0,0 +1,197 @@ +/** + * Who goes there — the school's community from the latest census plus GIAS + * facts. Benchmark chips use the computed state-school averages and must + * carry their provenance wording (never "England average" for computed + * figures). Copy verbatim from the reviewed mockups. + */ + +'use client'; + +import { verdict } from '@/lib/compareLogic'; +import type { Benchmarks, ComparisonData, School } from '@/lib/types'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +function pctSplit(part: number | null | undefined, total: number | null | undefined): string | null { + if (part == null || total == null || total === 0) return null; + return `${Math.round((part / total) * 100)}%`; +} + +export function CompareCommunity({ + schools, + data, + benchmarks, + isSecondary: propIsSecondary, +}: { + schools: School[]; + data: Record; + benchmarks?: Benchmarks; + isSecondary?: boolean; +}) { + const isSecondary = propIsSecondary !== undefined ? propIsSecondary : schools.some( + (school) => data[String(school.urn)]?.school_info?.attainment_8_score != null, + ); + const bench = isSecondary ? benchmarks?.secondary : benchmarks?.primary; + + const fsmChip = (value: number | null) => { + const anchor = bench?.fsm_pct ?? bench?.disadvantaged_pct ?? null; + if (value == null || anchor == null) return null; + const v = verdict(value, anchor, 3); + return ( + + {v === 'above' && `Above the state-school average (${Math.round(anchor)}%)`} + {v === 'close' && `About the state-school average (${Math.round(anchor)}%)`} + {v === 'below' && `Below the state-school average (${Math.round(anchor)}%)`} + + ); + }; + + return ( +
+ + + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info as (School & { gias_total_pupils?: number | null; capacity?: number | null }) | undefined; + const census = data[String(school.urn)]?.census; + const pupils = census?.total_pupils ?? info?.gias_total_pupils ?? null; + const capacity = info?.capacity ?? null; + let capNote: string | null = null; + if (pupils != null && capacity != null && capacity > 0) { + capNote = + pupils >= capacity + ? `${capacity.toLocaleString('en-GB')} places — at or above capacity` + : `of ${capacity.toLocaleString('en-GB')} places (${Math.round((pupils / capacity) * 100)}% full)`; + } + return ( + + {pupils != null ? ( + <> + {pupils.toLocaleString('en-GB')} + {capNote && {capNote}} + + ) : ( + No data + )} + + ); + })} + + + + {schools.map((school, i) => { + const census = data[String(school.urn)]?.census; + const girls = pctSplit(census?.female_pupils, census?.total_pupils); + const boys = pctSplit(census?.male_pupils, census?.total_pupils); + return ( + + {girls && boys ? `${girls} / ${boys}` : No data} + + ); + })} + + + + {schools.map((school, i) => { + const fsm = data[String(school.urn)]?.census?.fsm_pct ?? null; + return ( + + {fsm != null ? ( + <> + {Math.round(fsm)}% {fsmChip(fsm)} + + ) : ( + No data + )} + + ); + })} + + + + {schools.map((school, i) => { + const eal = data[String(school.urn)]?.census?.eal_pct ?? null; + return ( + + {eal != null ? `${Math.round(eal)}%` : No data} + + ); + })} + + + + {schools.map((school, i) => { + const rows = data[String(school.urn)]?.yearly_data ?? []; + let sen: number | null = null; + for (let r = rows.length - 1; r >= 0; r--) { + if (rows[r].sen_support_pct != null) { + sen = rows[r].sen_support_pct; + break; + } + } + const high = + sen != null && bench?.sen_support_pct != null && sen >= bench.sen_support_pct * 1.75; + return ( + + {sen != null ? ( + <> + {Math.round(sen)}% {high && Well above average} + + ) : ( + No data + )} + + ); + })} + + + + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info; + const faith = info?.religious_denomination; + const none = !faith || faith === 'Does not apply' || faith === 'None'; + return ( + + {none ? 'None' : faith} + + ); + })} + + + + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info; + return ( + + {info?.age_range || No data} + + ); + })} + + + + {schools.map((school, i) => { + const info = data[String(school.urn)]?.school_info; + const trust = info?.trust_name; + const la = info?.local_authority ?? school.local_authority; + return ( + + {trust ? trust : la ? `${la} council` : No data} + + ); + })} + + +
+ ); +} diff --git a/nextjs-app/components/compare/CompareOfsted.tsx b/nextjs-app/components/compare/CompareOfsted.tsx new file mode 100644 index 0000000..56ff359 --- /dev/null +++ b/nextjs-app/components/compare/CompareOfsted.tsx @@ -0,0 +1,237 @@ +/** + * Ofsted section — one visual grammar for inspection detail across all + * three regimes (legacy graded, interim carried-forward, renewed-framework + * report card). Copy comes verbatim from the reviewed mockups. + */ + +'use client'; + +import { + OFSTED_LEGACY_GRADES, + ofstedDisplay, + rcAreaLabel, + type OfstedDisplay, +} from '@/lib/compareLogic'; +import type { ComparisonData, OfstedInspection, School } from '@/lib/types'; +import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared'; + +const GRADE_TONE: Record = { + 1: 'good', + 2: 'good', + 3: 'warn', + 4: 'bad', +}; + +const RC_CODE_TONE = (code: number): 'good' | 'warn' | 'bad' | 'neutral' => + code <= 2 ? 'good' : code === 3 ? 'neutral' : code === 4 ? 'warn' : 'bad'; + +function formatInspectionDate(iso: string | null): string { + if (!iso) return '—'; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return '—'; + return d.toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); +} + +function yearsSince(iso: string | null): number | null { + if (!iso) return null; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return null; + return (Date.now() - d.getTime()) / (365.25 * 24 * 3600 * 1000); +} + +function ResultCell({ display }: { display: OfstedDisplay }) { + if (display.kind === 'none') { + return No inspection outcome in our dataset; + } + if (display.kind === 'report_card') { + return ( + <> + Report card + New-style inspection — no overall grade is given + + ); + } + if (display.kind === 'transitional') { + return ( + <> + + No overall grade + + + Inspected under transitional framework (sub-judgements only) + + + ); + } + return ( + <> + + {display.gradeLabel} + + + {display.carriedForward + ? 'Grade carried forward from an earlier inspection (ungraded visit since)' + : 'Overall grade (older-style inspection)'} + + + ); +} + +function JudgementDetailCell({ + ofsted, + display, + schoolName, +}: { + ofsted: OfstedInspection; + display: OfstedDisplay; + schoolName: string; +}) { + if (display.kind === 'report_card') { + const entries = Object.entries(ofsted.report_card ?? {}); + return ( +
+ {entries.map(([key, entry]) => ( +
+ {rcAreaLabel(key)} + {entry.label} +
+ ))} + {ofsted.rc_safeguarding_met != null && ( +
+ Safeguarding + + {ofsted.rc_safeguarding_met ? 'Met' : 'Not met'} + +
+ )} +
+ ); + } + + const legacyAreas: Array<[string, number | null]> = [ + ['Quality of education', ofsted.quality_of_education], + ['Behaviour & attitudes', ofsted.behaviour_attitudes], + ['Personal development', ofsted.personal_development], + ['Leadership & management', ofsted.leadership_management], + ['Early years provision', ofsted.early_years_provision], + ]; + const published = legacyAreas.filter(([, grade]) => grade != null); + + if (published.length === 0) { + return ( + + We don't hold area-by-area detail for this inspection — see {schoolName}'s + Ofsted page for the full report. + + ); + } + return ( +
+ {published.map(([label, grade]) => ( +
+ {label} + + {OFSTED_LEGACY_GRADES[grade as number] ?? String(grade)} + +
+ ))} +
+ ); +} + +export function CompareOfsted({ + schools, + data, +}: { + schools: School[]; + data: Record; +}) { + const displays = schools.map((school) => ofstedDisplay(data[String(school.urn)]?.ofsted)); + const kinds = new Set(displays.map((d) => d.kind).filter((k) => k !== 'none')); + const mixedRegimes = kinds.size > 1; + + return ( +
+ Ofsted is the schools inspectorate. It stopped giving a single overall grade in{' '} + September 2024; inspections between then and November 2025 kept the + area-by-area judgements without an overall grade, and from November 2025{' '} + new inspections produce a report card rating each area of school life on + a five-point scale. + {mixedRegimes && ( + <> A report card and an older overall grade aren't directly comparable. + )}{' '} + (Ofsted's "Expected standard" rating is unrelated to the KS2 "expected + standard" test measure further down this page.) + + } + > + + + {schools.map((school, i) => ( + + + + ))} + + + + {schools.map((school, i) => { + const ofsted = data[String(school.urn)]?.ofsted; + const age = yearsSince(ofsted?.inspection_date ?? null); + return ( + + {formatInspectionDate(ofsted?.inspection_date ?? null)}{' '} + {age != null && age > 4 && 4+ years ago} + + ); + })} + + + + + {schools.map((school, i) => { + const ofsted = data[String(school.urn)]?.ofsted; + return ( + + {ofsted ? ( + + ) : ( + No inspection in our dataset + )} + + ); + })} + + + + + {schools.map((school, i) => { + const url = + data[String(school.urn)]?.ofsted?.ofsted_page_url ?? + `https://reports.ofsted.gov.uk/provider/21/${school.urn}`; + return ( + + + {school.school_name}'s Ofsted page → + + + ); + })} + + +
+ ); +} diff --git a/nextjs-app/components/compare/TrendsExplorer.module.css b/nextjs-app/components/compare/TrendsExplorer.module.css new file mode 100644 index 0000000..7c74367 --- /dev/null +++ b/nextjs-app/components/compare/TrendsExplorer.module.css @@ -0,0 +1,77 @@ +.explore { + margin-top: 1rem; +} + +.explore summary { + cursor: pointer; + font-weight: 600; + color: var(--accent-coral-dark); + padding: 0.85rem 1.1rem; + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 8px; +} + +.explore[open] summary { + border-radius: 8px 8px 0 0; +} + +.inner { + border: 1px solid var(--border-light); + border-top: none; + border-radius: 0 0 8px 8px; + background: var(--bg-card); + padding: 1.25rem 1.5rem; +} + +.picker { + display: flex; + align-items: center; + gap: 0.6rem; + margin-bottom: 1rem; + flex-wrap: wrap; +} + +.picker label { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-secondary); +} + +.picker select { + font-family: inherit; + font-size: 0.9rem; + padding: 0.4rem 0.6rem; + border-radius: 8px; + border: 1px solid var(--border-light); + background: var(--bg-card); + color: var(--text-primary); + max-width: 100%; +} + +.desc { + font-size: 0.78rem; + color: var(--text-muted); +} + +.progressNote { + font-size: 0.8rem; + color: var(--text-muted); + margin: 0 0 1rem; +} + +/* ComparisonChart runs Chart.js with maintainAspectRatio:false, so it fills + its container's height — which must be *definite*. A min-height alone does + not resolve the chart wrapper's height:100%, leaving Chart.js to fall back + to its ~150px default (a squashed sliver). Give it a real height. */ +.chartBox { + height: 420px; +} + +@media (max-width: 640px) { + /* Taller on mobile: the mobile-only school chips sit above the canvas and + wrap to two rows for 3+ schools, so the plot keeps a usable height. */ + .chartBox { + height: 360px; + } +} diff --git a/nextjs-app/components/compare/TrendsExplorer.tsx b/nextjs-app/components/compare/TrendsExplorer.tsx new file mode 100644 index 0000000..0a5f83e --- /dev/null +++ b/nextjs-app/components/compare/TrendsExplorer.tsx @@ -0,0 +1,128 @@ +/** + * Explore trends — the full grouped metric catalogue (nothing from the old + * compare page is lost; spec §4's tier 3) driving the year-by-year chart with + * its England reference line. Matches the mockup: a measure picker and the + * chart only (no data table). + */ + +'use client'; + +import dynamic from 'next/dynamic'; + +import type { ComparisonData, MetricDefinition, NationalAverages, School } from '@/lib/types'; +import { track } from '@/lib/analytics'; +import { Section } from './sectionShared'; +import styles from './TrendsExplorer.module.css'; + +const ComparisonChart = dynamic( + () => import('../ComparisonChart').then((m) => m.ComparisonChart), + { ssr: false }, +); + +const PRIMARY_OPTGROUPS: { label: string; category: string }[] = [ + { label: 'Expected Standard', category: 'expected' }, + { label: 'Higher Standard', category: 'higher' }, + { label: 'Progress Scores', category: 'progress' }, + { label: 'Average Scores', category: 'average' }, + { label: 'Gender Performance', category: 'gender' }, + { label: 'Equity (Disadvantaged)', category: 'equity' }, + { label: 'School Context', category: 'context' }, + { label: 'Absence', category: 'absence' }, + { label: '3-Year Trends', category: 'trends' }, +]; + +const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [ + { label: 'GCSE Performance', category: 'gcse' }, +]; + +export const PRIMARY_CATEGORIES = PRIMARY_OPTGROUPS.map((g) => g.category); +export const SECONDARY_CATEGORIES = SECONDARY_OPTGROUPS.map((g) => g.category); + +export function TrendsExplorer({ + schools, + data, + metrics, + metric, + onMetricChange, + isPrimaryPhase, + nationalAverages, +}: { + schools: School[]; + data: Record; + metrics: MetricDefinition[]; + /** Controlled: the page owns the metric so the URL contract survives. */ + metric: string; + onMetricChange: (metric: string) => void; + isPrimaryPhase: boolean; + nationalAverages?: NationalAverages; +}) { + const allowedCategories = isPrimaryPhase ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES; + const optgroups = isPrimaryPhase ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS; + const filteredMetrics = metrics.filter((m) => allowedCategories.includes(m.category)); + const metricDef = metrics.find((m) => m.key === metric); + const metricLabel = metricDef?.label || metric; + + const nationalByYear: Record = {}; + for (const entry of nationalAverages?.by_year ?? []) { + const block = isPrimaryPhase ? entry.primary : entry.secondary; + nationalByYear[entry.year] = block?.[metric] ?? null; + } + + const handleMetricChange = (next: string) => { + track('compare_metric_changed', { metric: next, phase: isPrimaryPhase ? 'primary' : 'secondary' }); + onMetricChange(next); + }; + + return ( +
+
+ Year-by-year trends +
+
+ + + {metricDef?.description && {metricDef.description}} +
+ + {metric.includes('progress') && ( +

+ Progress scores measure pupils' progress from KS1 to KS2. A score of 0 equals the + national average. DfE stopped publishing KS2 progress after 2022/23 (no KS1 baseline). +

+ )} + +
+ +
+
+
+
+ ); +} diff --git a/nextjs-app/components/compare/compareSections.module.css b/nextjs-app/components/compare/compareSections.module.css new file mode 100644 index 0000000..f152a52 --- /dev/null +++ b/nextjs-app/components/compare/compareSections.module.css @@ -0,0 +1,264 @@ +/* Shared layout for the compare screen's measure-first sections. + Mobile base: each row-label becomes a measure header and each school cell + stacks under it (colour-coded via the cell's ::before school tag). + Desktop (≥761px): the mockups' grid — 200px row-label column + one column + per school (2–4 columns supported via --school-count). */ + +.section { + margin-top: 3rem; +} + +.sectionTitle { + font-family: var(--font-playfair), 'Playfair Display', Georgia, serif; + font-size: 1.45rem; + font-weight: 700; + margin: 0; + padding-left: 0.75rem; + border-left: 3px solid var(--accent-coral-dark); +} + +.how { + font-size: 0.85rem; + color: var(--text-muted); + margin: 0.35rem 0 0 0.95rem; + max-width: 70ch; +} + +.grid { + display: grid; + grid-template-columns: 1fr; + gap: 0; + margin-top: 1.25rem; +} + +/* Mobile base: each measure is a card; each cell is a school row led by a + colour dot + short name. `display: contents` at ≥761px dissolves the card + back into the shared grid. */ +.measure { + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 12px; + box-shadow: var(--shadow-soft); + padding: 0.75rem 0.85rem; + margin-bottom: 0.6rem; +} + +.rowLabel { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-primary); + display: flex; + align-items: center; + gap: 0.35rem; + padding: 0 0 0.1rem; +} + +.cell { + display: flex; + align-items: baseline; + gap: 0.35rem 0.5rem; + flex-wrap: wrap; + padding: 0.5rem 0; + border-top: 1px solid var(--border-light); + margin-top: 0.5rem; + font-size: 0.95rem; +} + +/* The school name gets its own full-width line above the value — real + school names are long and varied, so a fixed-width name column truncated + them ("Our Lady Queen of H…") or crowded the value. */ +.cellTag { + display: inline-flex; + align-items: center; + gap: 0.4rem; + flex-basis: 100%; + font-size: 0.8rem; + font-weight: 600; + color: var(--sc, var(--text-secondary)); + margin-bottom: 0.15rem; +} + +.cellDot { + width: 9px; + height: 9px; + border-radius: 50%; + background: var(--dot, var(--text-muted)); + flex: none; +} + +.big { + font-size: 1.05rem; + font-weight: 700; + font-variant-numeric: tabular-nums; +} + +.small { + display: block; + flex-basis: 100%; + font-size: 0.8rem; + color: var(--text-muted); + margin-top: 0; +} + +.chip { + display: inline-block; + font-size: 0.75rem; + font-weight: 600; + border-radius: 999px; + padding: 0.15rem 0.6rem; + white-space: nowrap; +} + +.chipGood { + background: rgba(45, 125, 125, 0.14); + color: var(--accent-teal); +} + +.chipWarn { + background: var(--accent-gold-bg); + color: var(--accent-gold-text); +} + +.chipBad { + background: var(--accent-coral-bg); + color: var(--accent-coral-dark); +} + +.chipNeutral { + background: var(--bg-secondary); + color: var(--text-secondary); +} + +.help { + display: inline-flex; + width: 15px; + height: 15px; + border-radius: 50%; + border: 1px solid var(--text-muted); + color: var(--text-muted); + font-size: 0.65rem; + align-items: center; + justify-content: center; + cursor: help; + flex: none; +} + +.badge { + display: inline-block; + font-weight: 700; + border-radius: 6px; + padding: 0.25rem 0.7rem; + font-size: 0.9rem; +} + +.badgeGood { + background: rgba(45, 125, 125, 0.14); + color: var(--accent-teal); +} + +.badgeWarn { + background: var(--accent-gold-bg); + color: var(--accent-gold-text); +} + +.badgeBad { + background: var(--accent-coral-bg); + color: var(--accent-coral-dark); +} + +.rcList { + display: flex; + flex-direction: column; + gap: 0.3rem; + margin-top: 0.2rem; +} + +.rcRow { + display: flex; + justify-content: space-between; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; +} + +.rcArea { + color: var(--text-secondary); +} + +.chipStack { + display: flex; + gap: 0.3rem; + flex-wrap: wrap; + flex-basis: 100%; + margin-top: 0.3rem; +} + +.barMini { + display: block; + height: 8px; + border-radius: 4px; + background: var(--bg-secondary); + overflow: hidden; + margin-top: 0.3rem; + max-width: 140px; +} + +.barMini > i { + display: block; + height: 100%; + border-radius: 4px; +} + +.card { + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 16px; + box-shadow: var(--shadow-soft); + padding: 1.25rem 1.5rem; + margin-top: 1rem; +} + +.link { + color: var(--accent-coral-dark); +} + +@media (min-width: 761px) { + .grid { + grid-template-columns: 200px repeat(var(--school-count, 3), 1fr); + gap: 0 0.75rem; + } + + /* Dissolve the per-measure card so its label + cells become grid items of + .grid, keeping columns aligned across every measure. */ + .measure { + display: contents; + } + + .cellTag { + display: none; + } + + .rowLabel { + color: var(--text-secondary); + padding: 0.85rem 0.5rem 0.85rem 0; + border-bottom: 1px solid var(--border-light); + } + + .cell { + display: block; + padding: 0.85rem 0.25rem; + border-top: none; + border-bottom: 1px solid var(--border-light); + margin-top: 0; + } + + .big { + font-size: 1.35rem; + } + + .small { + flex-basis: auto; + padding-left: 0; + margin-top: 0.1rem; + } +} diff --git a/nextjs-app/components/compare/sectionShared.tsx b/nextjs-app/components/compare/sectionShared.tsx new file mode 100644 index 0000000..0c27ea5 --- /dev/null +++ b/nextjs-app/components/compare/sectionShared.tsx @@ -0,0 +1,130 @@ +/** + * Small shared pieces for the compare sections: the section shell, the + * row-label + per-school-cell grid, and tone-mapped chips. Copy passed into + * these comes verbatim from the reviewed mockups + * (docs/superpowers/specs/mockups/) — do not paraphrase it here. + */ + +'use client'; + +import type { CSSProperties, ReactNode } from 'react'; + +import type { School } from '@/lib/types'; +import { CHART_COLORS, CHART_TEXT_COLORS, shortName } from '@/lib/utils'; +import styles from './compareSections.module.css'; + +export function Section({ + title, + how, + children, +}: { + title: string; + how?: ReactNode; + children: ReactNode; +}) { + return ( +
+

{title}

+ {how &&

{how}

} + {children} +
+ ); +} + +export function SectionGrid({ + schools, + children, +}: { + schools: School[]; + children: ReactNode; +}) { + return ( +
+ {children} +
+ ); +} + +export function RowLabel({ children, tip }: { children: ReactNode; tip?: string }) { + return ( +
+ {children} + {tip && ( + + ? + + )} +
+ ); +} + +/** + * One measure = its row label plus a cell per school. `display: contents` on + * desktop (see CSS) makes these flow into the section grid as if this wrapper + * weren't here, keeping columns aligned across measures; on mobile the wrapper + * becomes a card so each measure reads as its own block. + */ +export function Measure({ + label, + tip, + children, +}: { + label: ReactNode; + tip?: string; + children: ReactNode; +}) { + return ( +
+ {label} + {children} +
+ ); +} + +export function Cell({ + school, + index, + children, +}: { + school: School; + index: number; + children: ReactNode; +}) { + return ( +
+ {/* Mobile-only per-school tag (dot + short name); hidden on desktop, + where the column header identifies the school. */} + + + {children} +
+ ); +} + +export type ChipTone = 'good' | 'warn' | 'bad' | 'neutral'; + +const CHIP_TONE_CLASS: Record = { + good: styles.chipGood, + warn: styles.chipWarn, + bad: styles.chipBad, + neutral: styles.chipNeutral, +}; + +export function Chip({ tone, children }: { tone: ChipTone; children: ReactNode }) { + return {children}; +} + +export const sectionStyles = styles; diff --git a/nextjs-app/hooks/useComparison.ts b/nextjs-app/hooks/useComparison.ts index f7ed256..68913e8 100644 --- a/nextjs-app/hooks/useComparison.ts +++ b/nextjs-app/hooks/useComparison.ts @@ -1,50 +1,18 @@ /** - * Custom hook for managing school comparison state - * Uses shared context for real-time updates across components + * Custom hook for managing school comparison state. + * + * This hook is mounted on every page via the global Navigation and + * ComparisonToast, so it must stay cheap — it exposes basket state only. + * The compare page fetches `/api/compare` itself (ComparisonView); nothing + * ever read the comparison payload from here, so the previous per-page SWR + * fetch (which fired on every page whenever the basket was non-empty) was + * dead weight and has been removed. */ 'use client'; -import useSWR from 'swr'; -import { fetcher } from '@/lib/api'; import { useComparisonContext } from '@/context/ComparisonContext'; -import type { ComparisonResponse } from '@/lib/types'; export function useComparison() { - const { - selectedSchools, - addSchool, - removeSchool, - replaceSchools, - clearAll, - isSelected, - canAddMore, - isInitialized, - } = useComparisonContext(); - - // Fetch comparison data for selected schools - const urns = selectedSchools.map((s) => s.urn).join(','); - const { data, error, isLoading, mutate } = useSWR( - selectedSchools.length > 0 ? `/compare?urns=${urns}` : null, - fetcher, - { - revalidateOnFocus: false, - dedupingInterval: 10000, - } - ); - - return { - selectedSchools, - comparisonData: data?.comparison, - isLoading, - error, - addSchool, - removeSchool, - replaceSchools, - clearAll, - isSelected, - canAddMore, - isInitialized, - mutate, - }; + return useComparisonContext(); } diff --git a/nextjs-app/lib/compareChartData.ts b/nextjs-app/lib/compareChartData.ts new file mode 100644 index 0000000..728fd7a --- /dev/null +++ b/nextjs-app/lib/compareChartData.ts @@ -0,0 +1,102 @@ +/** + * Pure series-building for the comparison trend chart, extracted from + * ComparisonChart so it is unit-testable without a canvas. + * + * Chart truthfulness rules (spec §8.1): every academic year between the + * first and last data point appears on the axis — cancelled test years + * (2019/20, 2020/21) and the unpublished 2021/22 school-level year render + * as real gaps, never as compressed time; school lines never bridge gaps. + */ + +import type { ComparisonData } from './types'; + +/** 201819 → 201920 (academic-year arithmetic on YYYYYY codes). */ +function nextAcademicYear(year: number): number { + const start = Math.floor(year / 100); + const end = year % 100; + return (start + 1) * 100 + (end + 1); +} + +/** Every academic year from min(years) to max(years), inclusive. */ +export function fillAcademicYears(years: number[]): number[] { + if (years.length === 0) return []; + const ints = [...new Set(years.map((y) => Math.trunc(y)))].sort((a, b) => a - b); + const out: number[] = []; + let y = ints[0]; + const last = ints[ints.length - 1]; + while (y <= last && out.length < 50) { + out.push(y); + y = nextAcademicYear(y); + } + return out; +} + +export interface CompareChartSeries { + label: string; + data: Array; + /** Index into CHART_COLORS / point styles. */ + schoolIndex: number; + spanGaps: false; +} + +export interface EnglandSeries { + label: 'England average'; + data: Array; + borderDash: [number, number]; + spanGaps: false; +} + +export interface CompareChart { + years: number[]; + schoolDatasets: CompareChartSeries[]; + englandDataset: EnglandSeries | null; + /** True when England published a 2021/22 figure but no school has one — + * the UI shows: "DfE didn't publish school-level figures for 2021/22". */ + showUnpublished202122Note: boolean; +} + +export function buildCompareChart( + comparisonData: Record, + schools: Array<{ urn: number; school_name: string }>, + metric: string, + nationalByYear?: Record, +): CompareChart { + const rawYears = schools.flatMap( + (s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [], + ); + const years = fillAcademicYears(rawYears); + + const schoolDatasets: CompareChartSeries[] = schools.map((school, schoolIndex) => { + const rows = comparisonData[String(school.urn)]?.yearly_data ?? []; + const byYear = new Map>(); + for (const row of rows) byYear.set(Math.trunc(row.year), row as unknown as Record); + return { + label: school.school_name, + data: years.map((year) => { + const v = byYear.get(year)?.[metric]; + return typeof v === 'number' && !Number.isNaN(v) ? v : null; + }), + schoolIndex, + spanGaps: false, + }; + }); + + let englandDataset: EnglandSeries | null = null; + if (nationalByYear) { + const data = years.map((year) => { + const v = nationalByYear[year]; + return typeof v === 'number' && !Number.isNaN(v) ? v : null; + }); + if (data.some((v) => v != null)) { + englandDataset = { label: 'England average', data, borderDash: [5, 4], spanGaps: false }; + } + } + + const idx202122 = years.indexOf(202122); + const showUnpublished202122Note = + idx202122 >= 0 && + englandDataset?.data[idx202122] != null && + schoolDatasets.every((ds) => ds.data[idx202122] == null); + + return { years, schoolDatasets, englandDataset, showUnpublished202122Note }; +} diff --git a/nextjs-app/lib/compareLogic.ts b/nextjs-app/lib/compareLogic.ts new file mode 100644 index 0000000..b4cb44f --- /dev/null +++ b/nextjs-app/lib/compareLogic.ts @@ -0,0 +1,250 @@ +/** + * Comprehension rules for the compare screen, kept pure and unit-tested. + * + * These encode the expert-review requirements (spec §8 of the compare + * redesign): report-card summaries count graded areas only (safeguarding is + * a separate binary judgement), problem areas are always NAMED rather than + * folded into counts, grade labels pass through from the API (live-sampled + * Ofsted vocabulary — never invented here), admissions chips use one + * consistent metric, and progress bands follow DfE's confidence-interval + * methodology instead of thresholding point estimates. + */ + +import type { OfstedInspection, SchoolAdmissions } from './types'; + +// --------------------------------------------------------------------------- +// Verdicts against an anchor (England average or state-school benchmark) +// --------------------------------------------------------------------------- + +export type Verdict = 'above' | 'close' | 'below'; + +export function verdict(value: number, anchor: number, tolerance = 2): Verdict { + if (value >= anchor + tolerance) return 'above'; + if (value <= anchor - tolerance) return 'below'; + return 'close'; +} + +// --------------------------------------------------------------------------- +// Ofsted — three regimes, one display model +// --------------------------------------------------------------------------- + +export const OFSTED_LEGACY_GRADES: Record = { + 1: 'Outstanding', + 2: 'Good', + 3: 'Requires improvement', + 4: 'Inadequate', +}; + +/** rc_ key → the area label used across the reviewed mockups. */ +const RC_AREA_LABELS: Record = { + rc_inclusion: 'Inclusion', + rc_curriculum_teaching: 'Curriculum & teaching', + rc_achievement: 'Achievement', + rc_attendance_behaviour: 'Attendance & behaviour', + rc_personal_development: 'Personal development', + rc_leadership_governance: 'Leadership & governance', + rc_early_years: 'Early years', + rc_sixth_form: 'Sixth form', +}; + +export function rcAreaLabel(key: string): string { + return RC_AREA_LABELS[key] ?? key; +} + +export interface ReportCardSummary { + /** Graded areas only, grouped by label, best grade first. */ + counts: Array<{ label: string; count: number }>; + /** Areas rated Needs attention / Urgent improvement — always named. */ + problems: Array<{ areaLabel: string; label: string }>; + safeguarding: 'met' | 'not_met' | null; + /** True when every graded area is Expected standard or better and + * safeguarding is not "not met". */ + allClear: boolean; +} + +const PROBLEM_CODES = new Set([4, 5]); + +export function summariseReportCard(ofsted: OfstedInspection): ReportCardSummary { + const entries = Object.entries(ofsted.report_card ?? {}); + const byCode = new Map(); + const problems: ReportCardSummary['problems'] = []; + + for (const [key, entry] of entries) { + if (PROBLEM_CODES.has(entry.code)) { + problems.push({ areaLabel: rcAreaLabel(key), label: entry.label }); + } else { + const existing = byCode.get(entry.code); + if (existing) existing.count += 1; + else byCode.set(entry.code, { label: entry.label, count: 1 }); + } + } + + const counts = [...byCode.entries()] + .sort(([a], [b]) => a - b) + .map(([, v]) => v); + + const safeguarding = + ofsted.rc_safeguarding_met === true + ? 'met' + : ofsted.rc_safeguarding_met === false + ? 'not_met' + : null; + + return { + counts, + problems, + safeguarding, + allClear: entries.length > 0 && problems.length === 0 && safeguarding !== 'not_met', + }; +} + +export type OfstedDisplay = + | { kind: 'none' } + | { kind: 'graded'; grade: number; gradeLabel: string; carriedForward: false } + | { kind: 'carried_forward'; grade: number; gradeLabel: string; carriedForward: true } + | { kind: 'transitional' } + | { kind: 'report_card'; summary: ReportCardSummary }; + +export function ofstedDisplay( + ofsted: OfstedInspection | null | undefined, +): OfstedDisplay { + if (!ofsted) return { kind: 'none' }; + + // A report card is the newest inspection format; when present it wins — + // never derive or prefer an overall grade alongside it. + if (ofsted.report_card && Object.keys(ofsted.report_card).length > 0) { + return { kind: 'report_card', summary: summariseReportCard(ofsted) }; + } + + const grade = ofsted.overall_effectiveness; + const gradeLabel = grade != null ? OFSTED_LEGACY_GRADES[grade] : undefined; + if (grade == null || gradeLabel === undefined) { + if (ofsted.inspection_date) { + return { kind: 'transitional' }; + } + return { kind: 'none' }; + } + + if (ofsted.grade_source === 'ungraded_carried_forward') { + return { kind: 'carried_forward', grade, gradeLabel, carriedForward: true }; + } + return { kind: 'graded', grade, gradeLabel, carriedForward: false }; +} + +// --------------------------------------------------------------------------- +// Admissions — one consistent chip metric (first-preference success) +// --------------------------------------------------------------------------- + +export interface AdmissionsSummary { + firstPrefPct: number | null; + chip: { tone: 'good' | 'warn' | 'neutral'; text: string } | null; + /** e.g. "Named on 457 forms · 180 places" — total preferences at any rank, + * deliberately not phrased as head-to-head applications. */ + interest: string | null; +} + +export function summariseAdmissions( + a: SchoolAdmissions | null | undefined, +): AdmissionsSummary { + if (!a) return { firstPrefPct: null, chip: null, interest: null }; + + const pct = + a.first_preference_offer_pct != null + ? Math.round(a.first_preference_offer_pct) + : null; + + let chip: AdmissionsSummary['chip'] = null; + if (pct != null) { + if (pct >= 100) { + chip = { tone: 'good', text: 'All first choices offered' }; + } else if (pct < 75) { + chip = { tone: 'warn', text: 'Over 1 in 4 first choices missed out' }; + } else { + chip = { tone: pct >= 90 ? 'good' : 'neutral', text: `${pct}% of first choices offered` }; + } + } + + const interest = + a.total_applications != null && a.places_offered != null + ? `Named on ${a.total_applications.toLocaleString('en-GB')} forms · ${a.places_offered.toLocaleString('en-GB')} places` + : null; + + return { firstPrefPct: pct, chip, interest }; +} + +// --------------------------------------------------------------------------- +// Progress bands — DfE confidence-interval methodology +// --------------------------------------------------------------------------- + +export function progressBand( + score: number | null, + lower: number | null, + upper: number | null, +): 'above' | 'average' | 'below' | null { + if (score == null || lower == null || upper == null) return null; + if (lower > 0) return 'above'; + if (upper < 0) return 'below'; + return 'average'; +} + +// --------------------------------------------------------------------------- +// Dot-strip geometry +// --------------------------------------------------------------------------- + +export interface StripPoint { + /** 0–100 percentage position along the track. */ + pos: number; + labelAbove: boolean; + value: number; + schoolIndex: number; +} + +/** Labels within 4% of the domain of a lower neighbour flip above the strip + * (the reviewed mockups' collision nudge). */ +export function stripPositions( + values: Array, + min = 0, + max = 100, +): StripPoint[] { + const span = max - min; + const points = values + .map((value, schoolIndex) => ({ value, schoolIndex })) + .filter((p): p is { value: number; schoolIndex: number } => p.value != null) + .map((p) => ({ + value: p.value, + schoolIndex: p.schoolIndex, + pos: Math.min(100, Math.max(0, ((p.value - min) / span) * 100)), + labelAbove: false, + })); + + const nudge = span * 0.04; + let lastBelow = -Infinity; + for (const p of [...points].sort((a, b) => a.value - b.value)) { + if (p.value - lastBelow < nudge) { + p.labelAbove = true; + } else { + lastBelow = p.value; + } + } + return points; +} + +// --------------------------------------------------------------------------- +// Metric extraction +// --------------------------------------------------------------------------- + +/** Latest non-null yearly value of `metricKey` per school, in `urns` order. */ +export function latestValues( + data: Record }>, + urns: number[], + metricKey: string, +): Array { + return urns.map((urn) => { + const rows = data[String(urn)]?.yearly_data ?? []; + for (let i = rows.length - 1; i >= 0; i--) { + const v = (rows[i] as Record)[metricKey]; + if (typeof v === 'number' && !Number.isNaN(v)) return v; + } + return null; + }); +} diff --git a/nextjs-app/lib/types.ts b/nextjs-app/lib/types.ts index c29512c..f82d907 100644 --- a/nextjs-app/lib/types.ts +++ b/nextjs-app/lib/types.ts @@ -99,6 +99,21 @@ export interface OfstedInspection { rc_leadership_governance: number | null; rc_early_years: number | null; rc_sixth_form: number | null; + /** Where the effective overall grade came from: a graded (Section 5) + * inspection, or carried forward from an ungraded (Section 8) outcome. */ + grade_source?: 'graded' | 'ungraded_carried_forward' | null; + /** Renewed-framework (Nov 2025) area judgements, coded + labelled by the + * backend from the live-sampled Ofsted vocabulary. Empty when the school + * has no report-card inspection. Safeguarding is never included here. */ + report_card?: Record; + /** The school's page on ofsted.gov.uk (never a deep report link). */ + ofsted_page_url?: string; + report_url?: string | null; +} + +export interface ReportCardEntry { + code: number; + label: string; } export interface SchoolCensus { @@ -129,6 +144,12 @@ export interface SchoolAdmissions { /** 1st-preference applications per place offered (>1 means oversubscribed). */ oversubscription_ratio?: number | null; oversubscribed: boolean | null; + total_offers?: number | null; + second_preference_offers?: number | null; + third_preference_offers?: number | null; + /** Applications naming this school from families in another LA, and offers to them. */ + cross_la_applications?: number | null; + cross_la_offers?: number | null; } export interface SenDetail { @@ -172,6 +193,22 @@ export interface SchoolResult { school_id: number; year: number; + // Progress confidence intervals + writing working-towards (published for + // years with progress measures, i.e. up to 2022/23) + reading_progress_lower_ci?: number | null; + reading_progress_upper_ci?: number | null; + writing_progress_lower_ci?: number | null; + writing_progress_upper_ci?: number | null; + writing_working_towards_pct?: number | null; + maths_progress_lower_ci?: number | null; + maths_progress_upper_ci?: number | null; + + // KS4 banding and disadvantage gaps + /** DfE's own plain-English Progress 8 label, e.g. "Well above average". */ + progress_8_banding?: string | null; + attainment_8_disadvantage_gap?: number | null; + progress_8_disadvantage_gap?: number | null; + // Pupil numbers total_pupils: number | null; eligible_pupils: number | null; @@ -308,10 +345,48 @@ export interface SchoolDetailsResponse { export interface ComparisonData { school_info: School; yearly_data: SchoolResult[]; + // Supplementary blocks (additive; absent on an old backend) + ofsted?: OfstedInspection | null; + census?: SchoolCensus | null; + admissions?: SchoolAdmissions | null; + admissions_history?: SchoolAdmissions[]; + deprivation?: SchoolDeprivation | null; +} + +export interface BenchmarkBlock { + eal_pct: number | null; + sen_support_pct: number | null; + disadvantaged_pct: number | null; + fsm_pct?: number | null; + median_pupils: number | null; + /** Primary only — weighted by cohort size. */ + disadvantaged_rwm_expected_pct?: number | null; +} + +/** Computed from our dataset — NOT official DfE figures. UI copy must say + * "state-school average (computed from our dataset)" (the `source` string). */ +export interface Benchmarks { + source: string; + year: number; + primary: BenchmarkBlock; + secondary: BenchmarkBlock; +} + +export interface NationalAverages { + year: number; + primary: Record; + secondary: Record; + by_year: Array<{ + year: number; + primary: Record; + secondary: Record; + }>; } export interface ComparisonResponse { comparison: Record; + national_averages?: NationalAverages; + benchmarks?: Benchmarks; } export interface RankingItem { diff --git a/nextjs-app/lib/utils.ts b/nextjs-app/lib/utils.ts index fa1edf0..928ac74 100644 --- a/nextjs-app/lib/utils.ts +++ b/nextjs-app/lib/utils.ts @@ -59,6 +59,24 @@ export function truncate(text: string, maxLength: number): string { return text.slice(0, maxLength).trim() + '...'; } +/** + * A compact school label for tight spaces (mobile compare rows, chip bars): + * drop the trailing establishment-type words so "Barclay Primary School" → + * "Barclay", "St Mary's Catholic Primary School" → "St Mary's". Falls back to + * a length-capped truncation for names that don't carry a type suffix. + */ +export function shortName(name: string, maxLength = 32): string { + let s = name + .replace( + /\s+(primary|junior|infant|nursery|community|foundation|catholic|academy|school|college)\b.*$/i, + '', + ) + .trim(); + if (!s) s = name; + if (s.length > maxLength) s = s.slice(0, maxLength - 1).trim() + '…'; + return s; +} + /** * Format a school's age range for display, e.g. "3-11" → "Ages 3–11". * Display-only — leaves the raw `age_range` field (used for sixth-form diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index b8f85da..02addb1 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -38,6 +38,31 @@ default_args = { "retry_delay": timedelta(minutes=5), } +# The backend caches the marts DataFrame at startup; after any rebuild the +# cache must be invalidated or the API serves stale (or empty) data until the +# container restarts. +INVALIDATE_CACHE_CMD = """ +set -e +BACKEND_URL="${BACKEND_URL:-http://backend:80}" +ADMIN_KEY="${ADMIN_API_KEY:-changeme}" + +echo "Calling $BACKEND_URL/api/admin/reload ..." + +response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\ + --connect-timeout 10 --max-time 120 \\ + -X POST "$BACKEND_URL/api/admin/reload" \\ + -H "X-API-Key: $ADMIN_KEY" \\ + -H "Content-Type: application/json") + +echo "HTTP status: $response" +cat /tmp/reload_response.json + +if [ "$response" != "200" ]; then + echo "ERROR: backend cache reload failed (HTTP $response)" + exit 1 +fi +""" + # ── Daily DAG (GIAS + downstream) ────────────────────────────────────── @@ -91,7 +116,12 @@ print(f'Validation passed: {{count}} GIAS rows') bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py", ) - extract_group >> validate_raw >> dbt_build >> sync_typesense + invalidate_cache = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_group >> validate_raw >> dbt_build >> sync_typesense >> invalidate_cache # ── Monthly DAG (Ofsted) ─────────────────────────────────────────────── @@ -121,7 +151,12 @@ with DAG( bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py", ) - extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted + invalidate_cache_ofsted = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted >> invalidate_cache_ofsted # ── Annual DAG (EES: KS2, KS4, Census, Admissions) ─────────────────── @@ -153,7 +188,12 @@ with DAG( bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py", ) - extract_ees_group >> dbt_build_ees >> sync_typesense_ees + invalidate_cache_ees = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_ees_group >> dbt_build_ees >> sync_typesense_ees >> invalidate_cache_ees # ── Annual DAG (IDACI Deprivation) ──────────────────────────────────── @@ -178,4 +218,9 @@ with DAG( bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_idaci+ fact_deprivation+", ) - extract_idaci >> dbt_build_idaci + invalidate_cache_idaci = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_idaci >> dbt_build_idaci >> invalidate_cache_idaci diff --git a/pipeline/meltano.yml b/pipeline/meltano.yml index cfcda23..856ac86 100644 --- a/pipeline/meltano.yml +++ b/pipeline/meltano.yml @@ -49,6 +49,9 @@ plugins: - name: mi_url kind: string description: Ofsted Management Information download URL + - name: independent_mi_url + kind: string + description: Ofsted Independent Schools Management Information download URL - name: tap-uk-fbit namespace: uk_fbit diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index 168d88d..289e356 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import datetime import io import re @@ -14,20 +15,28 @@ GOV_UK_PAGE = ( "monthly-management-information-ofsteds-school-inspections-outcomes" ) +INDEPENDENT_GOV_UK_PAGE = ( + "https://www.gov.uk/government/statistical-data-sets/" + "non-association-independent-schools-inspections-and-outcomes-management-information" +) + # Column name → internal field, in priority order (first match wins). # Handles both current and older file formats. COLUMN_PRIORITY = { "urn": ["URN", "Urn", "urn"], "inspection_date": [ "Inspection start date of latest OEIF graded inspection", + "Inspection start date of latest OEIF standard inspection", "Inspection start date", "Inspection date", ], "inspection_type": [ "Inspection type of latest OEIF graded inspection", + "Inspection type of latest OEIF standard inspection", "Inspection type", ], "event_type_grouping": [ + "Event type grouping of latest OEIF standard inspection", "Event type grouping", "Inspection type grouping", ], @@ -52,10 +61,12 @@ COLUMN_PRIORITY = { "Effectiveness of leadership and management", ], "early_years_provision": [ + "Latest OEIF early years provision (where applicable)", "Latest OEIF early years provision", "Early years provision (where applicable)", ], "sixth_form_provision": [ + "Latest OEIF sixth form provision (where applicable)", "Latest OEIF sixth form provision", "Sixth form provision (where applicable)", ], @@ -68,6 +79,21 @@ COLUMN_PRIORITY = { "ungraded_inspection_date": [ "Date of latest ungraded inspection", ], + # Report Card fields (post-Nov 2025 framework). + "rc_safeguarding_met": ["Safeguarding standards"], + "rc_inclusion": ["Inclusion"], + "rc_curriculum_teaching": ["Curriculum and teaching"], + "rc_achievement": ["Achievement"], + "rc_attendance_behaviour": ["Attendance and behaviour"], + "rc_personal_development": ["Personal development and wellbeing"], + "rc_leadership_governance": ["Leadership and governance"], + "rc_early_years": ["Early years (where applicable)"], + "rc_sixth_form": ["Post-16 provision (where applicable)"], + "report_url": [ + "Web Link (opens in new window)", + "Web link to Ofsted provider page", + "Web link", + ], } @@ -90,6 +116,51 @@ def discover_csv_url() -> str | None: return matches[0] if matches else None +def discover_independent_csv_url() -> str | None: + """Scrape GOV.UK page to find the latest independent schools MI CSV download link.""" + resp = requests.get(INDEPENDENT_GOV_UK_PAGE, timeout=30) + resp.raise_for_status() + # Look for CSV attachment links + csv_links = re.findall( + r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.csv)"', + resp.text, + ) + if not csv_links: + # Fall back to ODS + csv_links = re.findall( + r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.ods)"', + resp.text, + ) + + months = { + 'january': 1, 'february': 2, 'march': 3, 'april': 4, 'may': 5, 'june': 6, + 'july': 7, 'august': 8, 'september': 9, 'october': 10, 'november': 11, 'december': 12 + } + + parsed_links = [] + for link in csv_links: + normalized_link = link.lower().replace('-', '_') + if 'most_recent' not in normalized_link: + continue + + match = re.search(r'as_at_(\d{1,2})_([a-z]+)_(\d{4})', normalized_link) + if match: + day, month_str, year = match.groups() + month = months.get(month_str) + if month: + try: + dt = datetime(int(year), month, int(day)) + parsed_links.append((dt, link)) + except ValueError: + continue + + parsed_links.sort(reverse=True) + if parsed_links: + return parsed_links[0][1] + + return csv_links[0] if csv_links else None + + class OfstedInspectionsStream(Stream): """Stream: Ofsted inspection records.""" @@ -111,6 +182,15 @@ class OfstedInspectionsStream(Stream): th.Property("sixth_form_provision", th.StringType), th.Property("ungraded_outcome", th.StringType), th.Property("ungraded_inspection_date", th.StringType), + th.Property("rc_safeguarding_met", th.StringType), + th.Property("rc_inclusion", th.StringType), + th.Property("rc_curriculum_teaching", th.StringType), + th.Property("rc_achievement", th.StringType), + th.Property("rc_attendance_behaviour", th.StringType), + th.Property("rc_personal_development", th.StringType), + th.Property("rc_leadership_governance", th.StringType), + th.Property("rc_early_years", th.StringType), + th.Property("rc_sixth_form", th.StringType), th.Property("report_url", th.StringType), ).to_dict() @@ -124,15 +204,8 @@ class OfstedInspectionsStream(Stream): break return mapping - def get_records(self, context): - import pandas as pd - - url = self.config.get("mi_url") or discover_csv_url() - if not url: - self.logger.error("Could not discover Ofsted MI download URL") - return - - self.logger.info("Downloading Ofsted MI: %s", url) + def _fetch_and_parse_url(self, url: str, pd) -> list[dict]: + """Download file and parse records.""" resp = requests.get(url, timeout=120) resp.raise_for_status() @@ -148,8 +221,6 @@ class OfstedInspectionsStream(Stream): lines = text.split("\n") header_idx = 0 for i, line in enumerate(lines[:20]): - # Match lines where URN appears as a CSV field (start or after comma), - # not as a substring of words like "turn" or "return". if re.search(r'(?:^|,)\s*URN\s*(?:,|$)', line): header_idx = i break @@ -167,16 +238,38 @@ class OfstedInspectionsStream(Stream): for _, row in df.iterrows(): record = {} for field, col in col_map.items(): - record[field] = row.get(col, None) + val = row.get(col, None) + if pd.isna(val): + val = None + record[field] = val # Cast URN try: - record["urn"] = int(record["urn"]) + record["urn"] = int(record.get("urn")) except (ValueError, KeyError, TypeError): continue yield record + def get_records(self, context): + import pandas as pd + + # 1. State-funded schools + state_url = self.config.get("mi_url") or discover_csv_url() + if state_url: + self.logger.info("Downloading Ofsted state-funded MI: %s", state_url) + yield from self._fetch_and_parse_url(state_url, pd) + else: + self.logger.error("Could not discover Ofsted state-funded MI download URL") + + # 2. Independent schools + ind_url = self.config.get("independent_mi_url") or discover_independent_csv_url() + if ind_url: + self.logger.info("Downloading Ofsted independent MI: %s", ind_url) + yield from self._fetch_and_parse_url(ind_url, pd) + else: + self.logger.error("Could not discover Ofsted independent MI download URL") + class TapUKOfsted(Tap): """Singer tap for UK Ofsted Management Information.""" @@ -185,6 +278,7 @@ class TapUKOfsted(Tap): config_jsonschema = th.PropertiesList( th.Property("mi_url", th.StringType, description="Direct URL to Ofsted MI file"), + th.Property("independent_mi_url", th.StringType, description="Direct URL to Ofsted Independent Schools MI file"), ).to_dict() def discover_streams(self): diff --git a/pipeline/scripts/diagnose_compare_gaps.py b/pipeline/scripts/diagnose_compare_gaps.py new file mode 100644 index 0000000..123ca99 --- /dev/null +++ b/pipeline/scripts/diagnose_compare_gaps.py @@ -0,0 +1,305 @@ +"""Diagnose the three data gaps blocking the compare-screen redesign. + +Run from repo root (network access required, no DB needed): + uv run --with singer-sdk --with pandas --with requests \ + python pipeline/scripts/diagnose_compare_gaps.py + +(singer_sdk is a transitive import of tap_uk_ees.tap / tap_uk_ofsted.tap and +is not part of the repo's default environment, hence the `uv run --with`.) +""" +import io +import re +import sys + +import pandas as pd +import requests + +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ees") +sys.path.insert(0, "pipeline/plugins/extractors/tap-uk-ofsted") +from tap_uk_ees.tap import ( # noqa: E402 + _KS2_NATIONAL_COL_MAP, + _KS2_NATIONAL_CSV_URL, + download_release_zip, + get_all_releases, +) +from tap_uk_ofsted.tap import discover_csv_url # noqa: E402 + +TIMEOUT = 120 + + +def check_national_gps_science(): + print("\n=== (a) National catalogue CSV: GPS/science columns ===") + resp = requests.get(_KS2_NATIONAL_CSV_URL, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False) + df.columns = [c.strip().lower() for c in df.columns] + for csv_col in ("pt_gps_exp", "pt_scita_exp", "avg_readscore", "avg_matscore", "avg_gpsscore"): + status = "PRESENT" if csv_col in df.columns else "MISSING" + print(f" {csv_col}: {status}") + gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c] + print(f" all gps/science-ish columns: {gps_like}") + if "geographic_level" in df.columns: + nat = df[df["geographic_level"].str.strip().str.lower() == "national"] + else: + print(" geographic_level column missing — cannot isolate national rows") + return + print(f" national rows time_periods: {sorted(nat['time_period'].unique())}") + # Sample the values our map would read for the latest year + latest = nat[nat["time_period"] == nat["time_period"].max()] + for csv_col, field in _KS2_NATIONAL_COL_MAP.items(): + val = latest.iloc[0].get(csv_col, "") if len(latest) else "" + print(f" {field} <- {csv_col} = {val!r}") + + +def check_ks2_attainment_years_subjects(): + print("\n=== (b) EES KS2 attainment: years & subject labels ===") + releases = get_all_releases("key-stage-2-attainment") + print(f" releases found: {[r['time_period'] for r in releases]}") + for release in releases: + try: + zf = download_release_zip(release["id"]) + except Exception as e: + print(f" {release['time_period']}: DOWNLOAD FAILED: {e}") + continue + name = next((n for n in zf.namelist() + if "ks2_school_attainment_data" in n and n.endswith(".csv")), None) + if not name: + print(f" {release['time_period']}: NO school attainment CSV in ZIP") + print(f" all CSVs in zip: {[n for n in zf.namelist() if n.endswith('.csv')]}") + continue + with zf.open(name) as f: + df = pd.read_csv(f, dtype=str, keep_default_na=False, nrows=200000) + years = sorted(df["time_period"].unique()) + subjects = sorted(df["subject"].unique()) + print(f" release {release['time_period']}: time_periods={years}") + print(f" subjects={subjects}") + + +def check_ofsted_report_card_columns(): + print("\n=== (c) Ofsted MI CSV: report-card columns ===") + url = discover_csv_url() + print(f" MI file: {url}") + if url is None or not url.lower().endswith(".csv"): + print(f" URL is not a CSV (likely ODS) — stopping this section. url={url!r}") + return + resp = requests.get(url, timeout=TIMEOUT) + resp.raise_for_status() + df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False, nrows=5) + rc_like = [c for c in df.columns + if re.search(r"report card|inclusion|curriculum|achievement|safeguard|well.?being|governance", c, re.I)] + print(f" candidate report-card columns ({len(rc_like)}):") + for c in rc_like: + print(f" - {c!r}") + print(f" all columns ({len(df.columns)}):") + for c in df.columns: + print(f" - {c!r}") + + +if __name__ == "__main__": + check_national_gps_science() + check_ks2_attainment_years_subjects() + check_ofsted_report_card_columns() + + +# FINDINGS 2026-07-12: run via +# uv run --with singer-sdk --with pandas --with requests \ +# python pipeline/scripts/diagnose_compare_gaps.py +# +# (a) National catalogue CSV (GPS/science) — NOT a source-data problem. +# pt_gps_exp, pt_scita_exp, avg_readscore, avg_matscore, avg_gpsscore are +# all PRESENT in the catalogue CSV and hold real numeric values for the +# latest national row (time_period 202425: pt_gps_exp='72.6' -> +# gps_expected_pct; pt_scita_exp='81.6' -> science_expected_pct). +# national time_periods present: 201516, 201617, 201718, 201819, 201920, +# 202021, 202122, 202223, 202324, 202425 (COVID years 201920/202021 are +# present as rows but suppressed with 'x' per the module docstring, not +# absent). So _KS2_NATIONAL_COL_MAP is correct and the extractor's own +# read of the source is fine end-to-end -- the NULLs in +# marts.fact_ks2_national_averages are NOT caused by a missing/renamed +# source column. The gap must be introduced downstream of the tap +# (staging/mart SQL, a stale/incomplete load, or a dbt model not +# selecting these two columns) -- Task 5/6 should look at the dbt +# staging model for ees_ks2_national and the mart definition, not the +# tap/column-map. +# +# (b) EES KS2 attainment (school-level, "key-stage-2-attainment" publication) +# releases found (via get_all_releases): [None, '202425', '202324', +# '202223', '202122']. The `None` entry is the *current/latest* release +# (its slug doesn't parse to a 6-digit time_period by _slug_to_time_period, +# but the CSV inside carries time_period='202425' -- same data as the +# 202425-labelled release). +# +# Only two of the four releases contain a school-level attainment CSV +# matching "ks2_school_attainment_data*.csv": +# - release None (latest): HAS IT -> time_periods=['202425'] +# subjects=['Grammar, punctuation and spelling', 'Maths', 'Reading', +# 'Reading, writing and maths', 'Science', 'Writing'] +# - release 202324: HAS IT -> time_periods=['202324'] +# subjects= same 6 labels as above +# - release 202223: NO school attainment CSV in ZIP. This +# release's ZIP instead contains only LA/regional/national/MAT-level +# files (e.g. ks2_regional_and_local_authority_*, ks2_multi_academy +# _trusts_*, ks2_national_*); no data/*school*attainment*.csv file +# exists at all in this release's package. This CONFIRMS the +# "subject-level 2022/23 is NULL in prod" symptom: the source +# release literally does not publish a school-level attainment file +# for 202223 under this filename pattern -- it's not a tap bug. +# - release 202122: NO school attainment CSV in ZIP. Same +# situation: ZIP has only LA/regional/national-level files (e.g. +# ks2_regional_and_local_authority_2016_to_2022_revised.csv, +# ks2_national_school_characteristics_2016_to_2022_revised.csv); +# no school-level attainment CSV present. This CONFIRMS "school-level +# 2021/22 is absent" -- again a genuine source-data absence, not an +# extractor bug. +# Implication for Tasks 5/6/7: 202122 and 202223 school-level attainment +# cannot be backfilled from the "key-stage-2-attainment" EES publication +# via this filename pattern -- those two years must either be sourced +# from a different EES dataset/file (e.g. one of the *_school_location_ +# and_pupil_characteristics or *_school_type_and_pupil_characteristics +# files present in those ZIPs, which may carry school-level rows under a +# different filename), left NULL with an explicit "source unavailable" +# note, or backfilled from the legacy DfE "Compare School Performance" +# wide-format CSVs referenced elsewhere in tap.py. Subject labels to use +# when a source *is* found for 202324/202425: +# 'Grammar, punctuation and spelling', 'Maths', 'Reading', +# 'Reading, writing and maths', 'Science', 'Writing' +# (Reading, writing and maths spans reading+writing+maths combined -- +# this is the RWM row.) +# +# (c) Ofsted MI CSV (report-card columns) — confirmed PRESENT. +# discover_csv_url() resolved to (as at run time, latest inspections +# 31 May 2026): +# https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/ +# Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv +# This is a real .csv (not .ods) so section (c) ran to completion. +# Exact report-card column headers (7 grade columns + their paired date +# columns, all present verbatim, case/spacing exactly as below): +# 'Safeguarding standards' / 'Safeguarding standards - date of grade' +# 'Inclusion' / 'Inclusion - date of grade' +# 'Curriculum and teaching' / 'Curriculum and teaching - date of grade' +# 'Achievement' / 'Achievement - date of grade' +# 'Attendance and behaviour' / 'Attendance and behaviour - date of grade' +# 'Personal development and wellbeing' / 'Personal development and wellbeing - date of grade' +# 'Leadership and governance' / 'Leadership and governance - date of grade' +# Plus a related pass/fail-style field: +# 'Latest OEIF safeguarding is effective?' (note: double space in the +# header, verbatim from source -- preserve exactly when mapping) +# These are the new-style "report card" single-word-area grades +# (introduced alongside the "Attendance and behaviour" split from +# "Personal development"); they coexist in the same CSV with the legacy +# 5-judgement OEIF columns ('Latest OEIF overall effectiveness', +# 'Latest OEIF quality of education', 'Latest OEIF behaviour and +# attitudes', 'Latest OEIF personal development', 'Latest OEIF +# effectiveness of leadership and management'). Task 7 should map the 7 +# report-card columns above (grade + date pairs, 6 of them, plus the +# safeguarding-effective flag) rather than inventing new column names. + +# TASK 6 VERIFICATION 2026-07-12: 2021/22 legacy KS2 school-level archive +# +# RESULT: BLOCKED at the source-data level. School-level KS2 attainment for +# academic year 2021/22 was never published anywhere publicly by DfE -- not +# in EES (confirmed by Task 1's finding (b) above), not in the legacy +# "Compare School Performance" download wizard, and not as a standalone +# performance-tables archive/ODS on assets.publishing.service.gov.uk. This +# is a deliberate DfE decision, not a gap in our extraction logic. +# +# Confirming quote (Key stage 2 attainment 2021/22 release notes, via +# https://explore-education-statistics.service.gov.uk/find-statistics/ +# key-stage-2-attainment/2021-22): +# "We will not publish key stage 2 data for academic year 2021/22 in +# performance tables (also known as Compare School and College +# Performance)." ... "The Department will, however, still produce the +# normal suite of key stage 2 accountability measures at school and +# multi-academy trust level and share these securely with primary +# schools, academy trusts and local authorities to inform school +# improvement discussions." +# (i.e. school-level 202122 KS2 results exist internally at DfE but were +# withheld from every public channel: performance tables/CSCP, EES, and by +# extension the legacy DfE archives the current legacy_ks2_urls entries in +# meltano.yml were sourced from.) +# +# What was tried: +# 1. Direct download URL pattern from the task brief: +# https://www.compare-school-performance.service.gov.uk/download-data?download=true®ions=0&filters=KS2&fileformat=csv&year=2021-2022&meta=false +# -> HTTP 404, HTML error page (not a CSV/ZIP). Saved response inspected; +# confirmed 404 via response headers (`content-type: text/html`). +# 2. Walked the actual multi-step download wizard at +# https://www.compare-school-performance.service.gov.uk/download-data +# with a browser User-Agent and a cookie jar, replicating the GET-based +# form steps: currentstep=year (downloadYear=2021-2022) -> currentstep= +# region (regiontype=all&la=0) -> currentstep=datatypes. On the final +# "datatypes" step, the checkbox list for 2021-2022 has NO "ks2" (or +# "ks2mats") option at all -- only ks4/ks4prov/ks4underlying/ks5* / +# pupil-destination/absence/census/mats checkboxes are present. +# Control check: repeating the same wizard walk for downloadYear= +# 2018-2019, 2022-2023 and 2023-2024 shows a "ks2" (and "ks2mats") +# checkbox present in all three; downloadYear=2020-2021 (COVID-cancelled +# KS2 SATs year) also has NO ks2 checkbox, matching the pattern for a +# year where school-level KS2 genuinely isn't published. 2021-2022 +# behaves identically to the cancelled 2020-2021 year, not like the +# normal 2018-2019/2022-2023/2023-2024 years. +# 3. Web search for a standalone KS2 2022 performance-tables archive +# (e.g. "england_ks2final" for 2022) on assets.publishing.service.gov.uk +# found no such file; only unrelated 2022/2023-dated documents. +# +# No ZIP was ever obtained -- /tmp/dfe-2021-2022-ks2.zip contains the 404 +# HTML error page from attempt (1) above, not a real archive. It contains +# no england_ks2final.csv (there is no ZIP to look inside). +# +# Column-map check (brief's Step 1): NOT RUN -- there is no 2021/22 +# england_ks2final.csv to check headers against. This is moot until/unless +# a non-public source (e.g. a manual/internal DfE extract) becomes +# available; _LEGACY_KS2_COLUMN_MAP itself is unchanged and untested here. +# +# Recommendation: mark 202122 school-level KS2 as a genuine, permanent +# source-data gap (not a backfill candidate) unless the project can obtain +# the internal DfE extract DfE says it shared "securely with primary +# schools, academy trusts and local authorities" -- that is not a route +# available to this pipeline. Task 6's meltano.yml change (Step 2) and the +# filebrowser upload should NOT proceed for 202122; there is nothing to +# upload. + +# TASK 7 VALUE SAMPLE 2026-07-12: live value_counts() over the 7 report-card +# columns (plus the related safeguarding-effective flag) in the same MI CSV +# resolved by discover_csv_url() as at run time (31 May 2026 inspections +# file). Blank cells read as the literal string 'NULL' (matches +# keep_default_na=False in tap.py). Observed non-blank values, verbatim: +# +# 'Safeguarding standards': 'Met' (1319), 'Not met' (10) +# 'Inclusion': 'Expected standard' (710), +# 'Strong standard' (447), 'Needs attention' (130), 'Exceptional' (23), +# 'Urgent improvement' (19) +# 'Curriculum and teaching': 'Expected standard' (797), +# 'Needs attention' (287), 'Strong standard' (206), +# 'Urgent improvement' (28), 'Exceptional' (11) +# 'Achievement': 'Expected standard' (701), +# 'Needs attention' (364), 'Strong standard' (207), +# 'Urgent improvement' (39), 'Exceptional' (18) +# 'Attendance and behaviour': 'Expected standard' (699), +# 'Strong standard' (405), 'Needs attention' (188), +# 'Urgent improvement' (21), 'Exceptional' (16) +# 'Personal development and wellbeing': 'Expected standard' (728), +# 'Strong standard' (504), 'Needs attention' (66), 'Exceptional' (23), +# 'Urgent improvement' (8) +# 'Leadership and governance': 'Expected standard' (813), +# 'Strong standard' (292), 'Needs attention' (172), +# 'Urgent improvement' (34), 'Exceptional' (18) +# 'Latest OEIF safeguarding is effective?' (note double space, not used by +# Task 7 -- kept for completeness): 'Yes' (12970), 'No' (96) +# +# So the 6 graded report-card columns share exactly one 5-value vocabulary: +# {'Exceptional', 'Strong standard', 'Expected standard', 'Needs attention', +# 'Urgent improvement'} -- no 'Attention needed' variant was observed +# anywhere, so parse_report_card_grade.sql does NOT need that speculative +# branch from the task brief. 'Safeguarding standards' is a separate +# two-value vocabulary {'Met', 'Not met'}. +# +# Collision check: 'Achievement' matches by EXACT list-membership +# (`candidate in df_columns`, a Python list containment check against the +# full column-name list, not a substring/regex match) against only +# ['Achievement', 'Achievement - date of grade'] -- the date-paired column +# has a different exact string and is never selected. Same check for +# 'Safeguarding standards' found only itself, its own date-of-grade column, +# and the unrelated 'Latest OEIF safeguarding is effective?' column (not +# mapped to any rc_* field). No legacy OEIF column is accidentally consumed +# by an rc_ mapping. diff --git a/pipeline/transform/macros/parse_report_card_grade.sql b/pipeline/transform/macros/parse_report_card_grade.sql new file mode 100644 index 0000000..2e1bfb5 --- /dev/null +++ b/pipeline/transform/macros/parse_report_card_grade.sql @@ -0,0 +1,17 @@ +-- Macro: Parse Ofsted Report Card grade (post-Nov 2025 framework) from text +-- into the 5-point scale. Real values confirmed via a live sample of the MI +-- CSV (see pipeline/scripts/diagnose_compare_gaps.py's +-- "TASK 7 VALUE SAMPLE 2026-07-12" note) -- unrecognised text (including the +-- 'NULL' sentinel used by the source CSV for blanks) parses to NULL, never +-- errors. + +{% macro parse_report_card_grade(column_name) %} + case lower(trim(nullif({{ column_name }}, 'NULL'))) + when 'exceptional' then 1 + when 'strong standard' then 2 + when 'expected standard' then 3 + when 'needs attention' then 4 + when 'urgent improvement' then 5 + else null + end +{% endmacro %} diff --git a/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql b/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql index 0fa7a72..c66bf5c 100644 --- a/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql +++ b/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql @@ -15,8 +15,11 @@ current_ks2 as ( year, total_pupils, eligible_pupils, rwm_expected_pct, rwm_high_pct, reading_expected_pct, reading_high_pct, reading_avg_score, reading_progress, + reading_progress_lower_ci, reading_progress_upper_ci, writing_expected_pct, writing_high_pct, writing_progress, + writing_progress_lower_ci, writing_progress_upper_ci, writing_working_towards_pct, maths_expected_pct, maths_high_pct, maths_avg_score, maths_progress, + maths_progress_lower_ci, maths_progress_upper_ci, gps_expected_pct, gps_high_pct, gps_avg_score, science_expected_pct, reading_absence_pct, writing_absence_pct, maths_absence_pct, gps_absence_pct, science_absence_pct, rwm_expected_boys_pct, rwm_high_boys_pct, rwm_expected_girls_pct, rwm_high_girls_pct, @@ -33,8 +36,11 @@ predecessor_ks2 as ( ks2.year, ks2.total_pupils, ks2.eligible_pupils, ks2.rwm_expected_pct, ks2.rwm_high_pct, ks2.reading_expected_pct, ks2.reading_high_pct, ks2.reading_avg_score, ks2.reading_progress, + ks2.reading_progress_lower_ci, ks2.reading_progress_upper_ci, ks2.writing_expected_pct, ks2.writing_high_pct, ks2.writing_progress, + ks2.writing_progress_lower_ci, ks2.writing_progress_upper_ci, ks2.writing_working_towards_pct, ks2.maths_expected_pct, ks2.maths_high_pct, ks2.maths_avg_score, ks2.maths_progress, + ks2.maths_progress_lower_ci, ks2.maths_progress_upper_ci, ks2.gps_expected_pct, ks2.gps_high_pct, ks2.gps_avg_score, ks2.science_expected_pct, ks2.reading_absence_pct, ks2.writing_absence_pct, ks2.maths_absence_pct, ks2.gps_absence_pct, ks2.science_absence_pct, ks2.rwm_expected_boys_pct, ks2.rwm_high_boys_pct, ks2.rwm_expected_girls_pct, ks2.rwm_high_girls_pct, diff --git a/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql b/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql index 4156d0e..9625ba7 100644 --- a/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql +++ b/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql @@ -18,7 +18,8 @@ current_ks4 as ( english_maths_strong_pass_pct, english_maths_standard_pass_pct, ebacc_entry_pct, ebacc_strong_pass_pct, ebacc_standard_pass_pct, ebacc_avg_score, gcse_grade_91_pct, - sen_pct, sen_support_pct, sen_ehcp_pct + sen_pct, sen_support_pct, sen_ehcp_pct, + progress_8_banding, attainment_8_disadvantage_gap, progress_8_disadvantage_gap from all_ks4 ), @@ -34,7 +35,8 @@ predecessor_ks4 as ( ks4.english_maths_strong_pass_pct, ks4.english_maths_standard_pass_pct, ks4.ebacc_entry_pct, ks4.ebacc_strong_pass_pct, ks4.ebacc_standard_pass_pct, ks4.ebacc_avg_score, ks4.gcse_grade_91_pct, - ks4.sen_pct, ks4.sen_support_pct, ks4.sen_ehcp_pct + ks4.sen_pct, ks4.sen_support_pct, ks4.sen_ehcp_pct, + ks4.progress_8_banding, ks4.attainment_8_disadvantage_gap, ks4.progress_8_disadvantage_gap from all_ks4 ks4 inner join {{ ref('int_school_lineage') }} lin on ks4.urn = lin.predecessor_urn diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 9a8832b..9fc7860 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -86,6 +86,13 @@ models: tests: [not_null] - name: year tests: [not_null] + - name: reading_progress_lower_ci + - name: reading_progress_upper_ci + - name: writing_progress_lower_ci + - name: writing_progress_upper_ci + - name: writing_working_towards_pct + - name: maths_progress_lower_ci + - name: maths_progress_upper_ci tests: - unique: column_name: "urn || '-' || year" @@ -97,6 +104,15 @@ models: tests: [not_null] - name: year tests: [not_null] + - name: progress_8_banding + tests: + - accepted_values: + values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average'] + config: + where: "progress_8_banding is not null" + severity: warn + - name: attainment_8_disadvantage_gap + - name: progress_8_disadvantage_gap tests: - unique: column_name: "urn || '-' || year" @@ -124,6 +140,11 @@ models: tests: [not_null] - name: year tests: [not_null] + - name: second_preference_offers + - name: third_preference_offers + - name: cross_la_applications + - name: cross_la_offers + - name: total_offers - name: fact_finance description: School financial data — one row per URN per year @@ -139,6 +160,12 @@ models: - name: year tests: [not_null, unique] + - name: fact_ks4_national_averages + description: Computed national KS4 averages (means across state schools in our dataset — not official DfE figures) — one row per academic year + columns: + - name: year + tests: [not_null, unique] + - name: fact_deprivation description: IDACI deprivation index — one row per URN columns: diff --git a/pipeline/transform/models/marts/fact_admissions.sql b/pipeline/transform/models/marts/fact_admissions.sql index fda217e..5956b81 100644 --- a/pipeline/transform/models/marts/fact_admissions.sql +++ b/pipeline/transform/models/marts/fact_admissions.sql @@ -5,9 +5,14 @@ select year, school_phase, places_offered, + total_offers, total_applications, first_preference_applications, first_preference_offers, + second_preference_offers, + third_preference_offers, + cross_la_applications, + cross_la_offers, first_preference_offer_pct, oversubscription_ratio, oversubscribed, diff --git a/pipeline/transform/models/marts/fact_ks2_performance.sql b/pipeline/transform/models/marts/fact_ks2_performance.sql index 290e94f..f361e91 100644 --- a/pipeline/transform/models/marts/fact_ks2_performance.sql +++ b/pipeline/transform/models/marts/fact_ks2_performance.sql @@ -15,13 +15,20 @@ select reading_high_pct, reading_avg_score, reading_progress, + reading_progress_lower_ci, + reading_progress_upper_ci, writing_expected_pct, writing_high_pct, writing_progress, + writing_progress_lower_ci, + writing_progress_upper_ci, + writing_working_towards_pct, maths_expected_pct, maths_high_pct, maths_avg_score, maths_progress, + maths_progress_lower_ci, + maths_progress_upper_ci, gps_expected_pct, gps_high_pct, gps_avg_score, diff --git a/pipeline/transform/models/marts/fact_ks4_national_averages.sql b/pipeline/transform/models/marts/fact_ks4_national_averages.sql new file mode 100644 index 0000000..5022332 --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks4_national_averages.sql @@ -0,0 +1,25 @@ +{{ config(materialized='table') }} + +-- Mart: Computed national KS4 averages — one row per academic year. +-- Unlike fact_ks2_national_averages (official DfE figures), DfE publishes no +-- KS4 national-headline dataset we ingest yet, so these are means computed +-- across the state schools in our dataset. Computed once at build time so the +-- API never has to aggregate the full performance table per request. +-- Semantics match the API's previous per-request computation: rows where +-- attainment_8_score is non-null; per-column means ignore NULLs. + +select + year, + round(avg(attainment_8_score)::numeric, 2) as attainment_8_score, + round(avg(progress_8_score)::numeric, 2) as progress_8_score, + round(avg(english_maths_standard_pass_pct)::numeric, 2) as english_maths_standard_pass_pct, + round(avg(english_maths_strong_pass_pct)::numeric, 2) as english_maths_strong_pass_pct, + round(avg(ebacc_entry_pct)::numeric, 2) as ebacc_entry_pct, + round(avg(ebacc_standard_pass_pct)::numeric, 2) as ebacc_standard_pass_pct, + round(avg(ebacc_strong_pass_pct)::numeric, 2) as ebacc_strong_pass_pct, + round(avg(ebacc_avg_score)::numeric, 2) as ebacc_avg_score, + round(avg(gcse_grade_91_pct)::numeric, 2) as gcse_grade_91_pct +from {{ ref('fact_ks4_performance') }} +where attainment_8_score is not null +group by year +order by year diff --git a/pipeline/transform/models/marts/fact_ks4_performance.sql b/pipeline/transform/models/marts/fact_ks4_performance.sql index 4607dfb..57de61b 100644 --- a/pipeline/transform/models/marts/fact_ks4_performance.sql +++ b/pipeline/transform/models/marts/fact_ks4_performance.sql @@ -16,6 +16,9 @@ select progress_8_score, progress_8_lower_ci, progress_8_upper_ci, + progress_8_banding, + attainment_8_disadvantage_gap, + progress_8_disadvantage_gap, progress_8_english, progress_8_maths, progress_8_ebacc, diff --git a/pipeline/transform/models/marts/fact_performance.sql b/pipeline/transform/models/marts/fact_performance.sql index 97a92ec..4b9c7b3 100644 --- a/pipeline/transform/models/marts/fact_performance.sql +++ b/pipeline/transform/models/marts/fact_performance.sql @@ -25,13 +25,20 @@ select ks2.reading_high_pct, ks2.reading_avg_score, ks2.reading_progress, + ks2.reading_progress_lower_ci, + ks2.reading_progress_upper_ci, ks2.writing_expected_pct, ks2.writing_high_pct, ks2.writing_progress, + ks2.writing_progress_lower_ci, + ks2.writing_progress_upper_ci, + ks2.writing_working_towards_pct, ks2.maths_expected_pct, ks2.maths_high_pct, ks2.maths_avg_score, ks2.maths_progress, + ks2.maths_progress_lower_ci, + ks2.maths_progress_upper_ci, ks2.gps_expected_pct, ks2.gps_high_pct, ks2.gps_avg_score, @@ -61,6 +68,9 @@ select ks4.progress_8_maths, ks4.progress_8_ebacc, ks4.progress_8_open, + ks4.progress_8_banding, + ks4.attainment_8_disadvantage_gap, + ks4.progress_8_disadvantage_gap, ks4.english_maths_strong_pass_pct, ks4.english_maths_standard_pass_pct, ks4.ebacc_entry_pct, diff --git a/pipeline/transform/models/staging/stg_ees_admissions.sql b/pipeline/transform/models/staging/stg_ees_admissions.sql index d1d98fc..4b6495d 100644 --- a/pipeline/transform/models/staging/stg_ees_admissions.sql +++ b/pipeline/transform/models/staging/stg_ees_admissions.sql @@ -32,6 +32,11 @@ renamed as ( {{ safe_numeric('times_put_as_any_preferred_school') }}::integer as total_applications, {{ safe_numeric('times_put_as_1st_preference') }}::integer as first_preference_applications, + -- Cross-borough demand: applications naming this school from families + -- living in another local authority, and offers made to them. + {{ safe_numeric('"all_applications_from_another_LA"') }}::integer as cross_la_applications, + {{ safe_numeric('"offers_to_applicants_from_another_LA"') }}::integer as cross_la_offers, + -- Proportions -- first_preference_offer_pct: of families who listed this school FIRST, -- the percentage that received an offer. 0–100 scale. diff --git a/pipeline/transform/models/staging/stg_ees_ks2.sql b/pipeline/transform/models/staging/stg_ees_ks2.sql index 7a5f375..9f50b19 100644 --- a/pipeline/transform/models/staging/stg_ees_ks2.sql +++ b/pipeline/transform/models/staging/stg_ees_ks2.sql @@ -39,6 +39,12 @@ pivoted as ( max(case when subject = 'Reading' and breakdown_topic = 'All pupils' and breakdown = 'Total' then {{ safe_numeric('progress_measure_score') }} end) as reading_progress, + max(case when subject = 'Reading' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as reading_progress_lower_ci, + max(case when subject = 'Reading' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as reading_progress_upper_ci, max(case when subject = 'Reading' and breakdown_topic = 'All pupils' and breakdown = 'Total' then {{ safe_numeric('absent_or_not_able_to_access_percent') }} end) as reading_absence_pct, @@ -53,6 +59,15 @@ pivoted as ( max(case when subject = 'Writing' and breakdown_topic = 'All pupils' and breakdown = 'Total' then {{ safe_numeric('progress_measure_score') }} end) as writing_progress, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as writing_progress_lower_ci, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as writing_progress_upper_ci, + max(case when subject = 'Writing' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('working_towards_expected_standard_pupil_percent') }} end) as writing_working_towards_pct, max(case when subject = 'Writing' and breakdown_topic = 'All pupils' and breakdown = 'Total' then {{ safe_numeric('absent_or_not_able_to_access_percent') }} end) as writing_absence_pct, @@ -70,6 +85,12 @@ pivoted as ( max(case when subject = 'Maths' and breakdown_topic = 'All pupils' and breakdown = 'Total' then {{ safe_numeric('progress_measure_score') }} end) as maths_progress, + max(case when subject = 'Maths' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as maths_progress_lower_ci, + max(case when subject = 'Maths' + and breakdown_topic = 'All pupils' and breakdown = 'Total' + then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as maths_progress_upper_ci, max(case when subject = 'Maths' and breakdown_topic = 'All pupils' and breakdown = 'Total' then {{ safe_numeric('absent_or_not_able_to_access_percent') }} end) as maths_absence_pct, @@ -143,13 +164,20 @@ select p.reading_high_pct, p.reading_avg_score, p.reading_progress, + p.reading_progress_lower_ci, + p.reading_progress_upper_ci, p.writing_expected_pct, p.writing_high_pct, p.writing_progress, + p.writing_progress_lower_ci, + p.writing_progress_upper_ci, + p.writing_working_towards_pct, p.maths_expected_pct, p.maths_high_pct, p.maths_avg_score, p.maths_progress, + p.maths_progress_lower_ci, + p.maths_progress_upper_ci, p.gps_expected_pct, p.gps_high_pct, p.gps_avg_score, diff --git a/pipeline/transform/models/staging/stg_ees_ks2_national.sql b/pipeline/transform/models/staging/stg_ees_ks2_national.sql index 7335ce6..44dfbe4 100644 --- a/pipeline/transform/models/staging/stg_ees_ks2_national.sql +++ b/pipeline/transform/models/staging/stg_ees_ks2_national.sql @@ -31,4 +31,10 @@ select from {{ source('raw', 'ees_ks2_national') }} where time_period ~ '^[0-9]+$' - and cast(trim(time_period) as integer) >= 201617 + -- 2015/16 was the first year of the current expected-standard tests, so it's + -- the correct floor (not 2016/17 -- that excluded a real, comparable national + -- row). GPS/science/scaled-score columns are already mapped correctly end to + -- end (tap.py's _KS2_NATIONAL_COL_MAP + this model select them fine); the + -- prod NULLs for those fields are stale raw.ees_ks2_national data from before + -- the map covered them, not a mapping bug -- no map change accompanies this fix. + and cast(trim(time_period) as integer) >= 201516 diff --git a/pipeline/transform/models/staging/stg_ees_ks4.sql b/pipeline/transform/models/staging/stg_ees_ks4.sql index 7cd1d3e..217b0bf 100644 --- a/pipeline/transform/models/staging/stg_ees_ks4.sql +++ b/pipeline/transform/models/staging/stg_ees_ks4.sql @@ -62,7 +62,16 @@ info as ( {{ safe_numeric('ks2_scaledscore_average') }} as prior_attainment_avg, {{ safe_numeric('sen_pupil_percent') }} as sen_pct, {{ safe_numeric('sen_with_ehcp_pupil_percent') }} as sen_ehcp_pct, - {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct + {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct, + -- EES suppression sentinels (z/c/x/q/u) and blanks must not reach the + -- mart as banding labels + case + when lower(trim(progress8_banding)) in ('', 'z', 'c', 'x', 'q', 'u', 'null') + then null + else trim(progress8_banding) + end as progress_8_banding, + {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap, + {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap from {{ source('raw', 'ees_ks4_info') }} where school_urn is not null ) @@ -102,7 +111,10 @@ select -- Context i.sen_pct, i.sen_ehcp_pct, - i.sen_support_pct + i.sen_support_pct, + i.progress_8_banding, + i.attainment_8_disadvantage_gap, + i.progress_8_disadvantage_gap from all_pupils p left join info i on p.urn = i.urn and p.year = i.year diff --git a/pipeline/transform/models/staging/stg_legacy_ks2.sql b/pipeline/transform/models/staging/stg_legacy_ks2.sql index c719f20..637f39a 100644 --- a/pipeline/transform/models/staging/stg_legacy_ks2.sql +++ b/pipeline/transform/models/staging/stg_legacy_ks2.sql @@ -17,13 +17,23 @@ select {{ safe_numeric('reading_high_pct') }} as reading_high_pct, {{ safe_numeric('reading_avg_score') }} as reading_avg_score, {{ safe_numeric('reading_progress') }} as reading_progress, + -- Progress CIs / working-towards: not published in the legacy CSVs. + -- Typed placeholders keep positional alignment with stg_ees_ks2 in + -- int_ks2_with_lineage's UNION ALL. + null::numeric as reading_progress_lower_ci, + null::numeric as reading_progress_upper_ci, {{ safe_numeric('writing_expected_pct') }} as writing_expected_pct, {{ safe_numeric('writing_high_pct') }} as writing_high_pct, {{ safe_numeric('writing_progress') }} as writing_progress, + null::numeric as writing_progress_lower_ci, + null::numeric as writing_progress_upper_ci, + null::numeric as writing_working_towards_pct, {{ safe_numeric('maths_expected_pct') }} as maths_expected_pct, {{ safe_numeric('maths_high_pct') }} as maths_high_pct, {{ safe_numeric('maths_avg_score') }} as maths_avg_score, {{ safe_numeric('maths_progress') }} as maths_progress, + null::numeric as maths_progress_lower_ci, + null::numeric as maths_progress_upper_ci, {{ safe_numeric('gps_expected_pct') }} as gps_expected_pct, {{ safe_numeric('gps_high_pct') }} as gps_high_pct, {{ safe_numeric('gps_avg_score') }} as gps_avg_score, diff --git a/pipeline/transform/models/staging/stg_legacy_ks4.sql b/pipeline/transform/models/staging/stg_legacy_ks4.sql index 2254070..9789926 100644 --- a/pipeline/transform/models/staging/stg_legacy_ks4.sql +++ b/pipeline/transform/models/staging/stg_legacy_ks4.sql @@ -41,8 +41,13 @@ select -- SEN null::numeric as sen_pct, + {{ safe_numeric('sen_ehcp_pct') }} as sen_ehcp_pct, {{ safe_numeric('sen_support_pct') }} as sen_support_pct, - {{ safe_numeric('sen_ehcp_pct') }} as sen_ehcp_pct + + -- Progress 8 banding & disadvantage gaps (not published in legacy format) + null::text as progress_8_banding, + null::numeric as attainment_8_disadvantage_gap, + null::numeric as progress_8_disadvantage_gap from {{ source('raw', 'legacy_ks4') }} where urn is not null diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_inspections.sql index 557d1bb..518b24a 100644 --- a/pipeline/transform/models/staging/stg_ofsted_inspections.sql +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.sql @@ -33,19 +33,23 @@ renamed as ( nullif(trim(ungraded_outcome), 'NULL') as ungraded_outcome, {{ parse_ungraded_outcome('ungraded_outcome') }}::integer as ungraded_grade, - -- Report Card fields (post-Nov 2025 framework) - -- TODO: add rc_* columns to tap-uk-ofsted schema once CSV column names are confirmed - null::text as rc_safeguarding_met, - null::text as rc_inclusion, - null::text as rc_curriculum_teaching, - null::text as rc_achievement, - null::text as rc_attendance_behaviour, - null::text as rc_personal_development, - null::text as rc_leadership_governance, - null::text as rc_early_years, - null::text as rc_sixth_form, + -- Report Card fields (post-Nov 2025 framework), 5-point scale: + -- 1 Exceptional · 2 Strong standard · 3 Expected standard + -- · 4 Needs attention · 5 Urgent improvement + case lower(trim(nullif(rc_safeguarding_met, 'NULL'))) + when 'met' then true + when 'not met' then false + end as rc_safeguarding_met, + {{ parse_report_card_grade('rc_inclusion') }}::integer as rc_inclusion, + {{ parse_report_card_grade('rc_curriculum_teaching') }}::integer as rc_curriculum_teaching, + {{ parse_report_card_grade('rc_achievement') }}::integer as rc_achievement, + {{ parse_report_card_grade('rc_attendance_behaviour') }}::integer as rc_attendance_behaviour, + {{ parse_report_card_grade('rc_personal_development') }}::integer as rc_personal_development, + {{ parse_report_card_grade('rc_leadership_governance') }}::integer as rc_leadership_governance, + {{ parse_report_card_grade('rc_early_years') }}::integer as rc_early_years, + {{ parse_report_card_grade('rc_sixth_form') }}::integer as rc_sixth_form, - report_url + nullif(trim(report_url), 'NULL') as report_url from source where urn is not null and (