docs: design for 2023/24 KS4/KS2 data and DfE LA averages (C2, H2)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:08:25 +01:00
11 changed files with 315 additions and 322 deletions
@@ -1273,63 +1273,17 @@ async def get_filter_options(request: Request):
    }
 def _la_averages_payload(df: pd.DataFrame) -> dict:
    """Per-LA Attainment 8 for the "vs LA avg" comparison: DfE's own LA
    averages (fact_ks4_la_averages, all state-funded schools), never a mean of
    the dataframe. That mean counted independent and special schools and put
    most LAs about 7 points low (audit H2).
    The year is the latest with any school Attainment 8, so the average and
    the scores set against it are the same year. Figures are keyed by our LA
    name through the LA code. An LA without a DfE figure for that year (City of
    London) is absent; no figures for the year, or no mart, give an empty map,
    so rows show no comparison, never another year's figure.
    """
    empty = {"year": 0, "secondary": {"attainment_8_by_la": {}}}
    if df.empty or "attainment_8_score" not in df.columns:
        return empty
    scored = df[df["attainment_8_score"].notna()]
    if scored.empty:
        return empty
    year = int(scored["year"].max())
    la = (df[["local_authority_code", "local_authority"]]
          .dropna()
          .drop_duplicates("local_authority_code"))
    name_by_code = {int(code): name for code, name in
                    zip(la["local_authority_code"], la["local_authority"])}
    from . import database
    from .models import Ks4LaAverage
    rows: list = []
    db = None
    try:
        db = database.SessionLocal()
        rows = db.query(Ks4LaAverage).filter(Ks4LaAverage.year == year).all()
    except Exception:
        import logging
        logging.getLogger(__name__).warning(
            "DfE LA averages unavailable for %s", year, exc_info=True)
        if db is not None:
            db.rollback()
    finally:
        if db is not None:
            db.close()
    by_la = {
        name_by_code[row.la_code]: row.attainment_8_score
        for row in rows
        if row.attainment_8_score is not None and row.la_code in name_by_code
    }
    return {"year": year, "secondary": {"attainment_8_by_la": by_la}}
@app.get("/api/la-averages")
@limiter.limit(f"{settings.rate_limit_per_minute}/minute")
 async def get_la_averages(request: Request):
-    """DfE's per-LA Attainment 8 averages for the latest year with results."""
+    """Get per-LA average Attainment 8 score for secondary schools in the latest year."""
-    return _la_averages_payload(load_school_data())
+    df = load_school_data()
    if df.empty:
        return {"year": 0, "secondary": {"attainment_8_by_la": {}}}
    latest_year = int(df["year"].max())
    sec_df = df[(df["year"] == latest_year) & df["attainment_8_score"].notna()]
    la_avg = sec_df.groupby("local_authority")["attainment_8_score"].mean().round(1).to_dict()
    return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}}
 _KS2_NATIONAL_METRICS = [
@@ -344,18 +344,6 @@ class Ks4NationalAverage(Base):
    gcse_grade_91_pct = Column(Float)
 class Ks4LaAverage(Base):
    """Official DfE KS4 local-authority averages (all state-funded schools) —
    one row per academic year and LA. la_code is the GIAS LA code."""
    __tablename__ = "fact_ks4_la_averages"
    __table_args__ = MARTS
    year = Column(Integer, primary_key=True)
    la_code = Column(Integer, primary_key=True)
    la_name = Column(String)
    attainment_8_score = Column(Float)
 class Ks2NationalAverage(Base):
    """Official DfE KS2 national headline averages — one row per academic year."""
    __tablename__ = "fact_ks2_national_averages"
@@ -1,119 +0,0 @@
 """/api/la-averages serves DfE's own LA averages (fact_ks4_la_averages).
 It used to average the dataframe: every school with an Attainment 8,
 independent and special schools included. Kensington and Chelsea came out at
 35.2 against DfE's 54.5, and most LAs about 7 points low (audit H2).
 """
 import numpy as np
 import pandas as pd
 import pytest
 from fastapi.testclient import TestClient
 LATEST = 202425
 def _df():
    return pd.DataFrame([
        # A state school and an independent: their mean, 37.65, is not DfE's figure.
        dict(year=LATEST, local_authority="Kensington and Chelsea", local_authority_code=207, attainment_8_score=54.9),
        dict(year=LATEST, local_authority="Kensington and Chelsea", local_authority_code=207, attainment_8_score=20.4),
        dict(year=LATEST, local_authority="Bristol, City of", local_authority_code=801, attainment_8_score=45.0),
        dict(year=LATEST, local_authority="West Sussex", local_authority_code=938, attainment_8_score=48.0),
        # DfE publishes no LA figure for City of London.
        dict(year=LATEST, local_authority="City of London", local_authority_code=201, attainment_8_score=30.0),
        # A newer year with primary results only.
        dict(year=202526, local_authority="Kensington and Chelsea", local_authority_code=207, attainment_8_score=np.nan),
    ])
 class _Row:
    def __init__(self, year, la_code, la_name, attainment_8_score):
        self.year = year
        self.la_code = la_code
        self.la_name = la_name
        self.attainment_8_score = attainment_8_score
 class _StubSession:
    rows = [
        _Row(LATEST, 207, "Kensington and Chelsea", 54.5),
        _Row(LATEST, 801, "Bristol City", 46.3),   # DfE's spelling, not ours
        _Row(LATEST, 938, "West Sussex", None),    # suppressed
        _Row(LATEST, 330, "Birmingham", 44.0),     # no school of ours there
        _Row(202324, 207, "Kensington and Chelsea", 54.5),
    ]
    def query(self, model):
        assert model.__name__ == "Ks4LaAverage"
        return self
    def filter(self, condition):
        # The payload filters on year == <year>; apply it as Postgres would.
        self._year = condition.right.value
        return self
    def all(self):
        return [r for r in self.rows if r.year == self._year]
    def rollback(self):
        pass
    def close(self):
        pass
 class _OldYearOnly(_StubSession):
    rows = [_Row(202324, 207, "Kensington and Chelsea", 54.5)]
 class _NoMart(_StubSession):
    def all(self):
        raise RuntimeError('relation "marts.fact_ks4_la_averages" does not exist')
@pytest.fixture()
 def payload(monkeypatch):
    from backend import app as app_module
    from backend import database as database_module
    def _run(session_cls):
        monkeypatch.setattr(database_module, "SessionLocal", session_cls)
        return app_module._la_averages_payload(_df())
    return _run
 def test_serves_dfe_figures_keyed_by_our_la_names(payload):
    out = payload(_StubSession)
    assert out["year"] == LATEST
    assert out["secondary"]["attainment_8_by_la"] == {
        "Kensington and Chelsea": 54.5,
        "Bristol, City of": 46.3,
    }
 def test_an_la_without_a_dfe_figure_is_absent(payload):
    by_la = payload(_StubSession)["secondary"]["attainment_8_by_la"]
    assert "City of London" not in by_la
    assert "West Sussex" not in by_la
    assert None not in by_la.values()
 def test_no_dfe_figures_for_the_year_give_an_empty_map(payload):
    assert payload(_OldYearOnly) == {"year": LATEST, "secondary": {"attainment_8_by_la": {}}}
 def test_a_missing_mart_gives_an_empty_map(payload):
    assert payload(_NoMart)["secondary"]["attainment_8_by_la"] == {}
 def test_the_endpoint_serves_dfe_figures(monkeypatch):
    from backend import app as app_module
    from backend import database as database_module
    monkeypatch.setattr(app_module, "load_school_data", _df)
    monkeypatch.setattr(database_module, "SessionLocal", _StubSession)
    resp = TestClient(app_module.app).get("/api/la-averages")
    assert resp.status_code == 200, resp.text
    assert resp.json()["secondary"]["attainment_8_by_la"]["Kensington and Chelsea"] == 54.5
@@ -0,0 +1,300 @@
 # 2023/24 Results and DfE LA Averages — Design
 **Date:** 2026-10-06
 **Status:** approved design, not yet implemented
 **Scope:** `tap_uk_ees` (KS4 results, KS4 information, new LA stream),
 `safe_numeric`, new `fact_ks4_la_averages` mart, annual EES DAG selector,
 `/api/la-averages`, secondary search rows and map cards
 **Fixes:** audit findings C2 and H2, from the 3 Oct 2026 accuracy audit
 ## Goal
 Show every 2023/24 result and school-information figure DfE published, and
 compare each state-funded secondary school with DfE's own local-authority
 average.
 ## The problem
 ### C2: 2023/24 is empty
 Bishop Stopford School (137086) shows Attainment 8 60.7 for 2022/23, nothing
 for 2023/24 and 58.7 for 2024/25. DfE's 2023/24 figures are Attainment 8 64.1,
 Progress 8 +1.02 and English and maths grade 4+ 91.7%. 2023/24 is the last year
 DfE published Progress 8 (2024/25 has no KS2 baseline), so the site shows no
 recent Progress 8 for any school. Site-wide, 4,170 listed schools have a DfE
 2023/24 Attainment 8 and 3,384 a Progress 8.
 There are three separate causes.
 1. **KS4 results.** The 2024/25 release's
   `202425_performance_tables_schools_final.csv` is a time series: it holds
   2022/23, 2023/24 and 2024/25 under the current column names, with the right
   values (Bishop Stopford 2023/24: 64.1, 1.02, 91.7). The 2023/24 release's own
   `202324_performance_tables_schools_final.csv`, re-issued on 10 March 2026,
   uses the older names (`t_pupils`, `avg_att8`, `avg_p8score`,
   `pt_l2basics_94` …). `EESDatasetStream` reads releases in the API's order,
   newest first, so the old file is read last. Its rows carry none of the
   declared fields, and target-postgres upserts on the stream's primary key
   (`append_only = not key_properties` in meltanolabs-target-postgres 0.8.0),
   so they overwrite the good 2023/24 rows with nulls.
   The two files share the keys of every "Total" row, but 34,254 of the 57,090
   2023/24 sub-group rows use different labels ("Low prior" against "Low prior
   attainment"). Those old-label rows sit in `raw.ees_ks4_performance` with
   null measures. Nothing reads them.
 2. **KS4 school information.** 2023/24 information exists only in the 2023/24
   release, in `202324_information_about_schools_final.csv`, with the older
   names. `EESKS4InfoStream` declares the newer ones, so prior attainment,
   SEN percentages, disadvantage gaps and Progress 8 banding are null for
   2023/24.
 3. **KS2 school information.** The 2023/24 file
   `ks2_school_information_data.csv` uses the declared names, and pupil counts
   load (school 147411: 818 pupils, 112 eligible). Its percentages are written
   with a sign (`ptfsm6cla1a = "34%"`). `safe_numeric` accepts only
   `^-?[0-9]+(\.[0-9]+)?$`, so disadvantaged, EAL, SEN and mobility percentages
   are null for every school in 2023/24.
 DfE published no school-level KS2 information file for 2022/23 (the 2023/24
 release's attainment file carries 2022/23 attainment rows only). Those nulls are
 a gap in the source, not a defect.
 ### H2: "vs LA avg" uses the wrong average
 `/api/la-averages` (`backend/app.py`) takes an unweighted mean of every school
 in the LA with an Attainment 8 score, independent and special schools included.
 Independent schools score low because DfE measures exclude IGCSEs, and special
 schools score low for other reasons, so the average is too low almost
 everywhere. Kensington and Chelsea's "LA avg" is 35.2: the mean of 6 state
 schools (54.9) and 8 independent schools (20.4). DfE's figure is 54.5. Of the
 151 LAs the audit compared, ours was lower in 147, by 7.1 points on average and
 by up to 19.3, so most secondary schools look better than their area.
 DfE's LA averages are already in the file the pipeline downloads for the
 England averages: the "summary, all state-funded" data set
 (`data-catalogue/data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv`). Its
 `Local authority` / `All state-funded` / Total rows match DfE's published
 performance-table LA averages (RECTYPE 4) for all 152 LAs in 2024/25, with no
 difference. `EESKs4NationalStream` keeps the England row and discards them.
 "All state-funded" is the same population as the England benchmark the site
 already shows.
 Computing the average ourselves from state-funded schools, weighted by pupils,
 was tested and rejected: it differs from DfE's figure by 1.1 points on average
 and is never exact.
 ## Non-goals
 - 2022/23 KS2 school information (DfE published none).
 - An "excludes IGCSEs" note wherever an independent school's Attainment 8
  appears. This change only stops comparing independent schools with the LA.
 - Showing 2023/24 Progress 8 in the GCSE section's headline. The history section
  shows it once the data loads; the 2024/25 banner stays true.
 - Updating the fixed EES data-set id when DfE publishes 2025/26. It already
  feeds the England averages; the LA stream shares it.
 - LA comparisons for other measures or on other pages.
 - Deleting the leftover old-label raw rows automatically.
 ## The rules
 ### The newest release owns every year it contains (KS4 results only)
 `EESDatasetStream` gets an opt-in class attribute,
 `_newest_release_owns_period: bool = False`. `EESKS4PerformanceStream` sets it
 to `True`. When it is on:
 - releases are processed newest first by `time_period` (from the release slug),
  not in the API's order;
 - the stream records each `time_period` it has emitted;
 - in each older release, rows whose `time_period` a newer release already
  emitted are dropped, and the stream logs how many it skipped and for which
  years;
 - years only an older release contains are emitted as before.
 The filter is a pure function, testable without a download. Other streams keep
 the current behaviour. A general rule would be wrong: the 2024/25 KS2 file holds
 98,448 of the 955,956 rows the 2023/24 release has for 2023/24.
 ### KS4 information: old names
 `EESKS4InfoStream._column_renames` maps the 2023/24 names onto the declared
 fields:
 | 2023/24 column | Declared field |
 |---|---|
 | `t_allks_pupils` | `allks_pupil_count` |
 | `t_allks_boys` | `allks_boys_count` |
 | `t_allks_girls` | `allks_girls_count` |
 | `t_pupils` | `endks4_pupil_count` |
 | `avg_ks2_scaledscore` | `ks2_scaledscore_average` |
 | `pt_sen_with_ehcp` | `sen_with_ehcp_pupil_percent` |
 | `pt_sen` | `sen_pupil_percent` |
 | `pt_sen_no_ehcp` | `sen_no_ehcp_pupil_percent` |
 | `diffn_att8` | `attainment8_diffn` |
 | `diffn_p8mea` | `progress8_diffn` |
 | `p8_banding` | `progress8_banding` |
 Newer files contain none of the old names, so they are unaffected.
 ### `safe_numeric` accepts a trailing `%`
 The pattern becomes `^-?[0-9]+(\.[0-9]+)?%?$` and the cast reads
 `rtrim(col, '%')`. A value such as `34%` can only mean 34. Suppression codes
 (`c`, `z`, `x` …) still become null.
 ### LA averages
 A new stream, `ees_ks4_la`, reads the same CSV as `ees_ks4_national` and keeps
 rows where `geographic_level = 'Local authority'`,
 `establishment_type_group = 'All state-funded'`, `breakdown_topic = 'Total'` and
 `breakdown = 'Total'` (case-insensitive, as the national stream compares). It
 emits `time_period`, `old_la_code`, `new_la_code`, `la_name` and the 8 headline
 measures the national stream emits (`_KS4_NATIONAL_COL_MAP`). Primary key:
 (`time_period`, `old_la_code`). The two streams share one download-and-filter
 helper.
 `old_la_code` is the GIAS LA code (`local_authority_code`), so schools join on
 the code, not the name. Names match today for every LA DfE publishes; DfE
 publishes no figure for City of London.
 ### Which schools get a gap
 The search row and the map card show "vs LA avg" only when the school has an
 Attainment 8 score, is neither special (`isSpecialSchool`) nor independent
 (`isIndependentSchool`: "independent" in the GIAS type, which covers "Other
 independent school" and "Other independent special school"), and its LA has a
 DfE figure for the year the endpoint serves.
 ## Delivery
 Two PRs, as for C1.
 ### PR 1: pipeline
 - `pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py`: release
  precedence, KS4 information renames, `ees_ks4_la` stream registered in
  `discover_streams`, shared national/LA helper.
 - `pipeline/transform/macros/safe_numeric.sql`: trailing `%`.
 - `pipeline/transform/models/staging/`: source `raw.ees_ks4_la` and
  `stg_ees_ks4_la` (view): `cast(old_la_code as integer) as la_code`,
  `cast(time_period as integer) as year`, `la_name`, measures via
  `safe_numeric`, named as in `stg_ees_ks4_national`.
 - `pipeline/transform/models/marts/fact_ks4_la_averages.sql` (table): one row
  per (`year`, `la_code`) with `la_name` and the columns of
  `fact_ks4_national_averages`. Schema tests: unique (`year`, `la_code`);
  `year`, `la_code` and `la_name` not null.
 - Data tests in `pipeline/transform/tests/`:
  - `assert_ks4_years_have_results`: every year in `stg_ees_ks4` has a non-null
    Attainment 8 for at least 50% of its rows. DfE's files reach 82% each year;
    2023/24 loads at 0% today. Pre-2019 years come from the legacy model and are
    not tested.
  - `assert_ks2_info_percentages_loaded`: for every year in `stg_ees_ks2` where
    at least 1,000 rows have `total_pupils`, at least 90% of those rows have
    `disadvantaged_pct`. DfE's files reach 96–97%. 2022/23 has no pupil counts
    and is skipped.
  - `assert_ks4_la_averages_cover_las`: the latest year in
    `fact_ks4_la_averages` has at least 145 LAs (DfE: 152).
 - `pipeline/dags/school_data_pipeline.py`: the annual EES build selects
  `stg_ees_ks4_la+`.
 - `docs/ARCHITECTURE.md`: LA averages come from DfE's data set.
 ### PR 2: backend and UI (after the EES DAG has run on PR 1)
 - `backend/models.py`: `Ks4LaAverage` for `marts.fact_ks4_la_averages`.
 - `backend/app.py` `/api/la-averages`: the year is the latest with any school
  Attainment 8 (as now). It reads that year's rows from the mart and keys each
  `attainment_8_score` by our LA name, through the `local_authority_code` →
  `local_authority` pairs in the school data. The response shape is unchanged.
  No rows for that year, a missing table or a query error give an empty map,
  logged, never another year's figures and never a computed mean.
 - `nextjs-app/lib/utils.ts`: `isIndependentSchool(school)`.
 - `nextjs-app/components/SecondarySchoolRow.tsx` and
  `nextjs-app/components/LeafletMapInner.tsx`: the rule in "Which schools get a
  gap".
 - `e2e/tests/journeys.spec.ts`: the two journeys under Testing.
 ## Testing
 **Extractor (pytest, `pipeline/tests/`, new):**
 - precedence: with the flag on, a year in a newer release is emitted once, from
  the newer release; a year only an older release has is kept; with the flag
  off, every row passes;
 - releases arriving oldest first are still processed newest first;
 - KS4 information: an old-format row yields `endks4_pupil_count`,
  `ks2_scaledscore_average`, `progress8_banding` and the rest of the table;
 - LA filter: a small CSV with national, regional, LA and sub-group rows yields
  one row per LA and year, with the declared fields.
 **dbt (local `pgserver`, as for C1):**
 - unit test on `stg_ees_ks2`: `34%` → 34, `34` → 34, `c` → null;
 - unit test on `stg_ees_ks4_la` or the mart: codes and years cast, measures
  carried;
 - the three data tests and the schema tests above;
 - `pipeline/tests/test_dag_selectors.py` passes with the new selector.
 **Backend (pytest):**
 - the response gives the mart's figure where the fixture's plain mean differs;
 - matching works by code when the mart's `la_name` differs from ours;
 - a mart without the served year, and a missing table, give an empty map.
 **Front end (Jest):**
 - `isIndependentSchool` for "Other independent school", "Other independent
  special school", "Academy converter" and null;
 - `SecondarySchoolRow` and the map card show no gap for an independent school
  and show one for a state school with an LA figure.
 **E2E (`journeys.spec.ts`, PR 2):**
 - C2: Bishop Stopford (137086) history shows 2023/24 Attainment 8 64.1 and
  Progress 8 +1.02. These are final figures, so the journey stays stable.
 - H2: a search returning an independent and a state secondary: the independent
  row has no "vs LA avg", the state row has one. The exact gap is not asserted:
  DfE's 2025/26 provisional KS4 data is due and would change it.
 ## Rollout and verification
 1. PR 1 merges to staging. Tudor runs `school_data_annual_ees` on staging.
 2. Through the staging API: Bishop Stopford 2023/24 Attainment 8 64.1 and
   Progress 8 1.02; school 147411 2023/24 disadvantaged 34; the LA mart has 152
   LAs for 2024/25. If a data test fails, the build stops before search sync;
   investigate before going further.
 3. PR 2 merges, so the post-merge E2E gate runs against loaded data.
 4. Production, Tudor's decision, in order: promote PR 1, run the EES DAG on
   production, promote PR 2. If PR 2 arrives first, the endpoint returns an
   empty map and rows show no gap, never a wrong one.
 5. Optional cleanup, in the PR 1 description for Tudor:
   `delete from raw.ees_ks4_performance where time_period = '202324' and pupil_count is null`.
   New-format rows always have `pupil_count` (suppressed values are `c` or `z`,
   not null), so this removes exactly the old-label leftovers.
 6. Validation on production against DfE's files: every listed school's 2023/24
   Attainment 8, Progress 8 and English and maths figures match; the 2023/24 KS4
   and KS2 information fields match; `/api/la-averages` equals DfE for all 152
   LAs; independent rows show no gap. Then mark C2 and H2 resolved in the audit
   report.
 ## Expected visible change
 - Secondary history charts and tables run unbroken from 2022/23 to 2024/25, and
  2023/24 Progress 8 appears for about 3,400 schools.
 - 2023/24 school information (KS2 and KS4) fills in.
 - "vs LA avg" drops for most state secondaries. In Kensington and Chelsea, a
  school with Attainment 8 60.0 moves from +24.8 to +5.5.
 - Independent schools and City of London schools show no LA gap.
 ## Risks
 - **Data-test thresholds stop a good build.** They were set from DfE's own
  files (82% and 96–97% against 50% and 90%). A failure means the data changed
  shape, which is what they are for.
 - **`safe_numeric` is shared by 14 models.** Values written as `n%` were null
  and become numbers. No DfE column uses `%` for anything but a percentage.
  The staging EES run is the check.
 - **Precedence drops data an older release holds more completely.** It is
  opt-in for KS4 results, where both files hold the same 57,090 2023/24 keys.
 - **The fixed EES data-set id goes stale** when 2025/26 is published. The
  endpoint's year check then gives no gap rather than a mismatched one.
@@ -417,15 +417,12 @@ test('a secondary search row compares its Attainment 8 with the LA average', asy
  // guards the comparison itself; the unit test pins the cache mode.
  const la = await (await page.request.get('/api/la-averages')).json();
  const averages: Record<string, number> = la.secondary?.attainment_8_by_la ?? {};
  // DfE publishes about 152 LA averages. An empty map (a missing mart, or a
  // year the LA data set has not reached) hides every comparison: fail, not skip.
  expect(Object.keys(averages).length).toBeGreaterThan(100);
  const res = await page.request.get('/api/schools?search=school&phase=secondary&page_size=50');
  expect(res.ok()).toBeTruthy();
  const school = ((await res.json()).schools ?? []).find(
    (s: { attainment_8_score?: number | null; local_authority?: string; school_type?: string }) =>
      s.attainment_8_score != null && s.local_authority != null && averages[s.local_authority] != null
-      && !/special|pupil referral|alternative provision|independent/i.test(s.school_type ?? ''));
+      && !/special|pupil referral|alternative provision/i.test(s.school_type ?? ''));
  test.skip(!school, 'no mainstream secondary with an LA average here');
  await searchByName(page, school.school_name);
@@ -436,52 +433,6 @@ test('a secondary search row compares its Attainment 8 with the LA average', asy
  await expect(stats.getByText(/vs LA avg/)).toBeVisible();
 });
 test('an independent secondary shows its Attainment 8 without an LA comparison', async ({ page }) => {
  // DfE's LA averages cover state-funded schools, and an independent school's
  // Attainment 8 leaves out IGCSEs, so a gap would mislead (audit H2). A state
  // school on the same page must show its gap first, so the absence is real.
  const la = await (await page.request.get('/api/la-averages')).json();
  const averages: Record<string, number> = la.secondary?.attainment_8_by_la ?? {};
  expect(Object.keys(averages).length).toBeGreaterThan(100);
  type Row = { urn: number; school_type?: string; local_authority?: string; attainment_8_score?: number | null };
  const compared = (s: Row) => s.attainment_8_score != null && s.local_authority != null
    && averages[s.local_authority] != null
    && !/special|pupil referral|alternative provision/i.test(s.school_type ?? '');
  let found: { la: string; state: Row; independent: Row } | null = null;
  for (const name of ['Kensington and Chelsea', 'Westminster', 'Camden', 'Hammersmith and Fulham', 'Barnet']) {
    // The search page asks for the same first 50 schools.
    const res = await page.request.get(`/api/schools?search=${encodeURIComponent(name)}&phase=secondary&page_size=50`);
    const schools: Row[] = ((await res.json()).schools ?? []).filter(compared);
    const independent = schools.find(s => /independent/i.test(s.school_type ?? ''));
    const state = schools.find(s => !/independent/i.test(s.school_type ?? ''));
    if (independent && state) { found = { la: name, state, independent }; break; }
  }
  test.skip(!found, 'no LA here lists a state and an independent secondary on one page');
  await page.goto(`/?search=${encodeURIComponent(found!.la)}&phase=secondary`);
  const stats = (urn: number) => page.locator(`a[href^="/school/${urn}-"]`).first()
    .locator('xpath=ancestor::div[contains(@class, "__rowContent")][1]')
    .locator('[class*="__line3"]');
  await expect(stats(found!.state.urn).getByText(/vs LA avg/)).toBeVisible({ timeout: 15_000 });
  const independent = stats(found!.independent.urn);
  await expect(independent.getByText(found!.independent.attainment_8_score!.toFixed(1))).toBeVisible();
  await expect(independent.getByText(/vs LA avg/)).toHaveCount(0);
 });
 test('a secondary shows its 2023/24 GCSE results, the last year DfE published Progress 8', async ({ page }) => {
  // DfE re-issued its 2023/24 file under older column names, and every
  // school's 2023/24 row loaded empty (audit C2). These are DfE's final
  // figures for Bishop Stopford School, so they do not change.
  await page.goto('/school/137086');
  const history = page.locator('#history');
  await history.getByText('View raw year-by-year data').click();
  const row = history.getByRole('row', { name: /2023\/24/ });
  await expect(row).toContainText('64.1');
  await expect(row).toContainText('+1.0');
  await expect(row).toContainText('91.7%');
 });
 test('a phase outside primary/secondary filters to that phase, not to everything', async ({ page }) => {
  // The search page offers every GIAS phase, but the API only knew the grouped
  // ones and silently dropped the rest — so "Nursery" returned primaries.
@@ -105,20 +105,3 @@ it('puts the card back when the pins are rebuilt for a reason other than the sch
  expect(container.querySelector('.sc-popup')).toHaveTextContent('Southmead Primary School');
  expect(container.querySelectorAll('.sc-pin--selected')).toHaveLength(1);
 });
 it('compares a state secondary with its LA average on the card, but not an independent one', () => {
  const state: School = { ...base, urn: 3, school_name: 'Holland Park School', school_type: 'Academy converter',
    phase: 'Secondary', local_authority: 'Kensington and Chelsea', attainment_8_score: 60,
    latitude: 51.5, longitude: -0.2, distance: 0.3 };
  const independent: School = { ...state, urn: 4, school_name: 'Abbey Gate College',
    school_type: 'Other independent school', attainment_8_score: 20.4 };
  const laAverages = { 'Kensington and Chelsea': 54.5 };
  const { container, rerender } = renderMap({ schools: [state, independent], laAverages, selectedUrn: 3 });
  expect(container.querySelector('.sc-popup')).toHaveTextContent('60.0 Att 8 +5.5 vs LA');
  rerender({ schools: [state, independent], laAverages, selectedUrn: 4 });
  const card = container.querySelector('.sc-popup')!;
  expect(card).toHaveTextContent('20.4 Att 8');
  expect(card).not.toHaveTextContent(/vs LA/);
 });
@@ -116,38 +116,3 @@ describe('SecondarySchoolRow shares the school page flags', () => {
    expect(screen.getByText('Fee-paying')).toBeInTheDocument();
  });
 });
 describe('SecondarySchoolRow LA comparison', () => {
  it('compares a state school with its LA average', () => {
    render(
      <SecondarySchoolRow
        school={{ ...base, school_type: 'Academy converter', attainment_8_score: 60 }}
        laAvgAttainment8={54.5}
      />,
    );
    expect(screen.getByText(/\+5\.5 vs LA avg/)).toBeInTheDocument();
  });
  it('keeps the comparison for a school whose type is unknown', () => {
    render(
      <SecondarySchoolRow
        school={{ ...base, school_type: null, attainment_8_score: 60 } as unknown as School}
        laAvgAttainment8={54.5}
      />,
    );
    expect(screen.getByText(/\+5\.5 vs LA avg/)).toBeInTheDocument();
  });
  it("shows an independent school's Attainment 8 without an LA comparison", () => {
    // DfE's LA average covers state-funded schools; an independent's
    // Attainment 8 leaves out IGCSEs (audit H2).
    render(
      <SecondarySchoolRow
        school={{ ...base, school_type: 'Other independent school', attainment_8_score: 20.4 }}
        laAvgAttainment8={54.5}
      />,
    );
    expect(screen.getByText('20.4')).toBeInTheDocument();
    expect(screen.queryByText(/vs LA avg/)).not.toBeInTheDocument();
  });
 });
@@ -391,20 +391,3 @@ describe('singleSexLabel', () => {
    expect(singleSexLabel(undefined)).toBeNull();
  });
 });
 describe('isIndependentSchool', () => {
  const { isIndependentSchool } = require('@/lib/utils');
  it('matches both GIAS independent types', () => {
    expect(isIndependentSchool({ school_type: 'Other independent school' })).toBe(true);
    expect(isIndependentSchool({ school_type: 'Other independent special school' })).toBe(true);
  });
  it('does not match state-funded types or a missing type', () => {
    for (const t of ['Academy converter', 'Community school', 'Free schools', 'Non-maintained special school']) {
      expect(isIndependentSchool({ school_type: t })).toBe(false);
    }
    expect(isIndependentSchool({ school_type: null })).toBe(false);
    expect(isIndependentSchool({})).toBe(false);
  });
 });
@@ -9,7 +9,7 @@ import { useEffect, useRef, useState } from 'react';
 import L from 'leaflet';
 import 'leaflet/dist/leaflet.css';
 import type { School } from '@/lib/types';
-import { schoolUrl, isSpecialSchool, isIndependentSchool, buildOfstedListBadge, listRwmValue } from '@/lib/utils';
+import { schoolUrl, isSpecialSchool, buildOfstedListBadge, listRwmValue } from '@/lib/utils';
 interface LeafletMapInnerProps {
  schools: School[];
@@ -70,7 +70,7 @@ function metricHtml(school: School, { nationalAvgRwm, laAverages }: CardContext)
    const score = school.attainment_8_score;
    const laAvg = school.local_authority ? (laAverages?.[school.local_authority] ?? null) : null;
    let delta = '';
-    if (!special && !isIndependentSchool(school) && laAvg != null) {
+    if (!special && laAvg != null) {
      const diff = Math.round((score - laAvg) * 10) / 10;
      // Att8 runs 0–90 in 0.1 steps; ±0.5 is meaningful, where RWM needs ±2.
      const cls = diff >= 0.5 ? 'sc-up' : diff <= -0.5 ? 'sc-down' : '';
@@ -11,7 +11,7 @@
 'use client';
 import type { School } from '@/lib/types';
-import { buildOfstedListBadge, getPhaseStyle, schoolUrl, formatAgeRange, isProposedToClose, isSpecialSchool, isIndependentSchool } from '@/lib/utils';
+import { buildOfstedListBadge, getPhaseStyle, schoolUrl, formatAgeRange, isProposedToClose, isSpecialSchool } from '@/lib/utils';
 import { schoolFlags, schoolTypeLabel } from '@/lib/schoolFacts';
 import styles from './SecondarySchoolRow.module.css';
@@ -45,12 +45,10 @@ export function SecondarySchoolRow({
  const att8 = school.attainment_8_score;
  // The school's own Attainment 8 is a same-school figure — shown whenever it
  // exists (special schools included; their type tag on line 2 gives context).
-  // Only the vs-LA-average delta, a benchmark comparison, is dropped: for
+  // Only the vs-LA-average delta, a benchmark comparison, is dropped for
-  // special schools / PRUs / AP, whose pupils aren't measured against it
+  // special schools / PRUs / AP, whose pupils aren't measured against it fairly.
  // fairly, and for independent schools, because DfE's LA average covers
  // state-funded schools and an independent's Attainment 8 leaves out IGCSEs.
  const laDelta =
-    att8 != null && !isSpecialSchool(school) && !isIndependentSchool(school) && laAvgAttainment8 != null
+    att8 != null && !isSpecialSchool(school) && laAvgAttainment8 != null
      ? att8 - laAvgAttainment8
      : null;
@@ -758,16 +758,6 @@ export function isSpecialSchool(school: { school_type?: string | null }): boolea
  return /\bspecial\b/.test(t) || /pupil referral/.test(t) || /alternative provision/.test(t);
 }
 /**
 * Independent (fee-paying) schools: GIAS types "Other independent school" and
 * "Other independent special school". DfE's Attainment 8 for them leaves out
 * IGCSEs, and DfE's LA averages cover state-funded schools only, so callers
 * drop the "vs LA avg" comparison for them (audit H2).
 */
 export function isIndependentSchool(school: { school_type?: string | null }): boolean {
  return /\bindependent\b/i.test(school.school_type ?? '');
 }
 /**
 * Whether GIAS records a religious character. "None", "Does not apply" and
 * "Not applicable" are the register's ways of saying it has none; the place