feat(api): compare endpoint carries supplementary blocks, national averages and benchmarks

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB

backend/app.py

@@ -25,6 +25,7 @@ import asyncio
 from .config import settings
 from .data_loader import (
     clear_cache,
+    compute_benchmarks,
     load_school_data,
     load_latest_school_data,
     geocode_single_postcode,
@@ -662,6 +663,34 @@ async def compare_schools(
     if comparison_data.empty:
         raise HTTPException(status_code=404, detail="No schools found")
 
+    # One session for all schools' supplementary blocks; failures degrade
+    # to empty blocks rather than failing a working comparison (mirrors
+    # the detail endpoint's defensive pattern).
+    from . import database
+
+    _EMPTY_SUPPLEMENTARY = {
+        "ofsted": None,
+        "census": None,
+        "admissions": None,
+        "admissions_history": [],
+        "deprivation": None,
+    }
+    supplementary_by_urn: dict = {}
+    db = None
+    try:
+        db = database.SessionLocal()
+        for urn in urn_list:
+            supp = get_supplementary_data(db, urn)
+            supplementary_by_urn[urn] = {
+                key: supp.get(key, default)
+                for key, default in _EMPTY_SUPPLEMENTARY.items()
+            }
+    except Exception:
+        supplementary_by_urn = {}
+    finally:
+        if db is not None:
+            db.close()
+
     result = {}
     for urn in urn_list:
         school_data = comparison_data[comparison_data["urn"] == urn].sort_values("year")
@@ -679,9 +708,16 @@ async def compare_schools(
                     "rwm_expected_pct": float(latest["rwm_expected_pct"]) if pd.notna(latest.get("rwm_expected_pct")) else None,
                 },
                 "yearly_data": clean_for_json(school_data),
+                **supplementary_by_urn.get(urn, dict(_EMPTY_SUPPLEMENTARY)),
             }
 
-    return {"comparison": result}
+    return {
+        "comparison": result,
+        # Official DfE anchors + computed state-school benchmarks so the
+        # compare UI can label provenance correctly (spec §8.6).
+        "national_averages": _national_averages_payload(df),
+        "benchmarks": compute_benchmarks(df),
+    }
 
 
 @app.get("/api/filters")
@@ -727,22 +763,17 @@ async def get_la_averages(request: Request):
     return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}}
 
 
-@app.get("/api/national-averages")
+def _national_averages_payload(df: pd.DataFrame) -> dict:
-@limiter.limit(f"{settings.rate_limit_per_minute}/minute")
+    """National-averages payload shared by /api/national-averages and
-async def get_national_averages(request: Request):
+    /api/compare. Official DfE KS2 figures come from the mart table;
-    """
+    KS4 figures are computed from our dataset (no DfE dataset yet)."""
-    Compute national average for each metric from the latest data year.
-    Returns separate averages for primary (KS2) and secondary (KS4) schools.
-    Values are derived from the loaded DataFrame so they automatically
-    stay current when new data is loaded.
-    """
-    df = load_school_data()
     if df.empty:
         return {"primary": {}, "secondary": {}}
 
     ks2_metrics = [
         "rwm_expected_pct", "rwm_high_pct",
         "reading_expected_pct", "writing_expected_pct", "maths_expected_pct",
+        "gps_expected_pct", "gps_high_pct", "science_expected_pct",
         "reading_avg_score", "maths_avg_score", "gps_avg_score",
         "reading_progress", "writing_progress", "maths_progress",
         "overall_absence_pct", "persistent_absence_pct",
@@ -777,12 +808,13 @@ async def get_national_averages(request: Request):
 
     # Per-year KS2 primary averages: use official DfE figures from the mart table.
     # Per-year KS4 secondary averages: computed from our dataset (no DfE dataset yet).
-    from .database import SessionLocal
+    from . import database
     from .models import Ks2NationalAverage
 
     by_year = []
+    db = None
     try:
-        db = SessionLocal()
+        db = database.SessionLocal()
         nat_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all()
         # Build a lookup of computed secondary averages per year as fallback
         secondary_by_year = {}
@@ -810,6 +842,7 @@ async def get_national_averages(request: Request):
                 "secondary": secondary_by_year.get(yr, {}),
             })
     finally:
+        if db is not None:
             db.close()
 
     # Update latest_primary with official DfE figure for the latest year if available
@@ -826,6 +859,17 @@ async def get_national_averages(request: Request):
     }
 
 
+@app.get("/api/national-averages")
+@limiter.limit(f"{settings.rate_limit_per_minute}/minute")
+async def get_national_averages(request: Request):
+    """
+    National averages: official DfE KS2 figures per year plus computed
+    KS4 averages, derived from the loaded DataFrame and the
+    fact_ks2_national_averages mart.
+    """
+    return _national_averages_payload(load_school_data())
+
+
 @app.get("/api/metrics")
 @limiter.limit(f"{settings.rate_limit_per_minute}/minute")
 async def get_available_metrics(request: Request):

backend/tests/test_compare_enrichment.py

@@ -0,0 +1,121 @@
+"""/api/compare enrichment for the compare redesign: per-school
+supplementary blocks, top-level national_averages (shared with the
+/api/national-averages endpoint) and computed benchmarks — all additive."""
+
+import types
+
+import numpy as np
+import pandas as pd
+import pytest
+from fastapi.testclient import TestClient
+
+LATEST = 202425
+
+CANNED_SUPPLEMENTARY = {
+    "ofsted": {"overall_effectiveness": 2, "grade_source": "graded",
+               "report_card": {}, "ofsted_page_url": "https://reports.ofsted.gov.uk/provider/21/100140"},
+    "census": {"year": 202526, "fsm_pct": 29.8},
+    "admissions": {"year": 202627, "second_preference_offers": 4},
+    "admissions_history": [{"year": 202627, "second_preference_offers": 4}],
+    "sen_detail": None,
+    "phonics": None,
+    "deprivation": {"idaci_decile": 4},
+    "finance": None,
+}
+
+
+def _two_primary_schools_df() -> pd.DataFrame:
+    rows = []
+    for urn, name, rwm, dis in ((100140, "Plumcroft Primary School", 79.0, 72.0),
+                                (138690, "Barclay Primary School", 87.0, 86.0)):
+        rows.append(dict(
+            urn=urn, school_name=name, local_authority="Greenwich",
+            school_type="Community school", address="1 Road", phase="Primary",
+            year=LATEST, rwm_expected_pct=rwm, attainment_8_score=np.nan,
+            eligible_pupils=60, rwm_expected_disadvantaged_pct=dis,
+            eal_pct=20.0, sen_support_pct=14.0, disadvantaged_pct=25.0,
+            total_pupils=1000.0,
+        ))
+    return pd.DataFrame(rows)
+
+
+class _StubNatRow:
+    year = 202425
+    rwm_expected_pct = 62.1
+    gps_expected_pct = 72.0
+    science_expected_pct = 81.0
+
+
+class _StubSession:
+    def query(self, *a, **k):
+        return self
+
+    def order_by(self, *a, **k):
+        return self
+
+    def all(self):
+        return [_StubNatRow()]
+
+    def close(self):
+        pass
+
+
+@pytest.fixture()
+def client(monkeypatch):
+    from backend import app as app_module
+    from backend import database as database_module
+
+    monkeypatch.setattr(app_module, "load_school_data", _two_primary_schools_df)
+    monkeypatch.setattr(
+        app_module, "get_supplementary_data", lambda db, urn: dict(CANNED_SUPPLEMENTARY)
+    )
+    monkeypatch.setattr(database_module, "SessionLocal", _StubSession)
+    return TestClient(app_module.app, raise_server_exceptions=False)
+
+
+def test_existing_shape_is_preserved(client):
+    body = client.get("/api/compare?urns=100140,138690").json()
+    school = body["comparison"]["100140"]
+    assert school["school_info"]["rwm_expected_pct"] == 79.0
+    assert school["yearly_data"][0]["year"] == LATEST
+
+
+def test_each_school_gains_supplementary_blocks(client):
+    body = client.get("/api/compare?urns=100140,138690").json()
+    for urn in ("100140", "138690"):
+        school = body["comparison"][urn]
+        assert school["ofsted"]["grade_source"] == "graded"
+        assert school["census"]["fsm_pct"] == 29.8
+        assert school["admissions"]["second_preference_offers"] == 4
+        assert school["admissions_history"][0]["year"] == 202627
+        assert school["deprivation"]["idaci_decile"] == 4
+
+
+def test_top_level_national_averages_and_benchmarks(client):
+    body = client.get("/api/compare?urns=100140,138690").json()
+    assert body["national_averages"]["year"] == LATEST
+    assert body["benchmarks"]["source"] == "state-school average (computed from our dataset)"
+    # weighted over equal cohorts of 72 and 86 = 79.0
+    assert body["benchmarks"]["primary"]["disadvantaged_rwm_expected_pct"] == 79.0
+
+
+def test_supplementary_failure_degrades_not_500(client, monkeypatch):
+    from backend import app as app_module
+
+    def _boom(db, urn):
+        raise RuntimeError("marts unavailable")
+
+    monkeypatch.setattr(app_module, "get_supplementary_data", _boom)
+    resp = client.get("/api/compare?urns=100140")
+    assert resp.status_code == 200
+    school = resp.json()["comparison"]["100140"]
+    assert school["ofsted"] is None
+    assert school["admissions_history"] == []
+
+
+def test_national_averages_endpoint_exposes_gps_science(client):
+    body = client.get("/api/national-averages").json()
+    latest_primary_by_year = [e["primary"] for e in body["by_year"] if e["primary"]]
+    assert latest_primary_by_year, "expected official by_year rows from the stub"
+    assert latest_primary_by_year[-1]["gps_expected_pct"] == 72.0
+    assert latest_primary_by_year[-1]["science_expected_pct"] == 81.0