docs: design for 2023/24 KS4/KS2 data and DfE LA averages (C2, H2)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app.py

@@ -1273,63 +1273,17 @@ async def get_filter_options(request: Request):
     }
 
 
-def _la_averages_payload(df: pd.DataFrame) -> dict:
-    """Per-LA Attainment 8 for the "vs LA avg" comparison: DfE's own LA
-    averages (fact_ks4_la_averages, all state-funded schools), never a mean of
-    the dataframe. That mean counted independent and special schools and put
-    most LAs about 7 points low (audit H2).
-
-    The year is the latest with any school Attainment 8, so the average and
-    the scores set against it are the same year. Figures are keyed by our LA
-    name through the LA code. An LA without a DfE figure for that year (City of
-    London) is absent; no figures for the year, or no mart, give an empty map,
-    so rows show no comparison, never another year's figure.
-    """
-    empty = {"year": 0, "secondary": {"attainment_8_by_la": {}}}
-    if df.empty or "attainment_8_score" not in df.columns:
-        return empty
-    scored = df[df["attainment_8_score"].notna()]
-    if scored.empty:
-        return empty
-    year = int(scored["year"].max())
-
-    la = (df[["local_authority_code", "local_authority"]]
-          .dropna()
-          .drop_duplicates("local_authority_code"))
-    name_by_code = {int(code): name for code, name in
-                    zip(la["local_authority_code"], la["local_authority"])}
-
-    from . import database
-    from .models import Ks4LaAverage
-
-    rows: list = []
-    db = None
-    try:
-        db = database.SessionLocal()
-        rows = db.query(Ks4LaAverage).filter(Ks4LaAverage.year == year).all()
-    except Exception:
-        import logging
-        logging.getLogger(__name__).warning(
-            "DfE LA averages unavailable for %s", year, exc_info=True)
-        if db is not None:
-            db.rollback()
-    finally:
-        if db is not None:
-            db.close()
-
-    by_la = {
-        name_by_code[row.la_code]: row.attainment_8_score
-        for row in rows
-        if row.attainment_8_score is not None and row.la_code in name_by_code
-    }
-    return {"year": year, "secondary": {"attainment_8_by_la": by_la}}
-
-
 @app.get("/api/la-averages")
 @limiter.limit(f"{settings.rate_limit_per_minute}/minute")
 async def get_la_averages(request: Request):
-    """DfE's per-LA Attainment 8 averages for the latest year with results."""
-    return _la_averages_payload(load_school_data())
+    """Get per-LA average Attainment 8 score for secondary schools in the latest year."""
+    df = load_school_data()
+    if df.empty:
+        return {"year": 0, "secondary": {"attainment_8_by_la": {}}}
+    latest_year = int(df["year"].max())
+    sec_df = df[(df["year"] == latest_year) & df["attainment_8_score"].notna()]
+    la_avg = sec_df.groupby("local_authority")["attainment_8_score"].mean().round(1).to_dict()
+    return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}}
 
 
 _KS2_NATIONAL_METRICS = [

backend/models.py

@@ -344,18 +344,6 @@ class Ks4NationalAverage(Base):
     gcse_grade_91_pct = Column(Float)
 
 
-class Ks4LaAverage(Base):
-    """Official DfE KS4 local-authority averages (all state-funded schools) —
-    one row per academic year and LA. la_code is the GIAS LA code."""
-    __tablename__ = "fact_ks4_la_averages"
-    __table_args__ = MARTS
-
-    year = Column(Integer, primary_key=True)
-    la_code = Column(Integer, primary_key=True)
-    la_name = Column(String)
-    attainment_8_score = Column(Float)
-
-
 class Ks2NationalAverage(Base):
     """Official DfE KS2 national headline averages — one row per academic year."""
     __tablename__ = "fact_ks2_national_averages"

backend/tests/test_la_averages_marts.py

@@ -1,119 +0,0 @@
-"""/api/la-averages serves DfE's own LA averages (fact_ks4_la_averages).
-
-It used to average the dataframe: every school with an Attainment 8,
-independent and special schools included. Kensington and Chelsea came out at
-35.2 against DfE's 54.5, and most LAs about 7 points low (audit H2).
-"""
-
-import numpy as np
-import pandas as pd
-import pytest
-from fastapi.testclient import TestClient
-
-LATEST = 202425
-
-
-def _df():
-    return pd.DataFrame([
-        # A state school and an independent: their mean, 37.65, is not DfE's figure.
-        dict(year=LATEST, local_authority="Kensington and Chelsea", local_authority_code=207, attainment_8_score=54.9),
-        dict(year=LATEST, local_authority="Kensington and Chelsea", local_authority_code=207, attainment_8_score=20.4),
-        dict(year=LATEST, local_authority="Bristol, City of", local_authority_code=801, attainment_8_score=45.0),
-        dict(year=LATEST, local_authority="West Sussex", local_authority_code=938, attainment_8_score=48.0),
-        # DfE publishes no LA figure for City of London.
-        dict(year=LATEST, local_authority="City of London", local_authority_code=201, attainment_8_score=30.0),
-        # A newer year with primary results only.
-        dict(year=202526, local_authority="Kensington and Chelsea", local_authority_code=207, attainment_8_score=np.nan),
-    ])
-
-
-class _Row:
-    def __init__(self, year, la_code, la_name, attainment_8_score):
-        self.year = year
-        self.la_code = la_code
-        self.la_name = la_name
-        self.attainment_8_score = attainment_8_score
-
-
-class _StubSession:
-    rows = [
-        _Row(LATEST, 207, "Kensington and Chelsea", 54.5),
-        _Row(LATEST, 801, "Bristol City", 46.3),   # DfE's spelling, not ours
-        _Row(LATEST, 938, "West Sussex", None),    # suppressed
-        _Row(LATEST, 330, "Birmingham", 44.0),     # no school of ours there
-        _Row(202324, 207, "Kensington and Chelsea", 54.5),
-    ]
-
-    def query(self, model):
-        assert model.__name__ == "Ks4LaAverage"
-        return self
-
-    def filter(self, condition):
-        # The payload filters on year == <year>; apply it as Postgres would.
-        self._year = condition.right.value
-        return self
-
-    def all(self):
-        return [r for r in self.rows if r.year == self._year]
-
-    def rollback(self):
-        pass
-
-    def close(self):
-        pass
-
-
-class _OldYearOnly(_StubSession):
-    rows = [_Row(202324, 207, "Kensington and Chelsea", 54.5)]
-
-
-class _NoMart(_StubSession):
-    def all(self):
-        raise RuntimeError('relation "marts.fact_ks4_la_averages" does not exist')
-
-
-@pytest.fixture()
-def payload(monkeypatch):
-    from backend import app as app_module
-    from backend import database as database_module
-
-    def _run(session_cls):
-        monkeypatch.setattr(database_module, "SessionLocal", session_cls)
-        return app_module._la_averages_payload(_df())
-
-    return _run
-
-
-def test_serves_dfe_figures_keyed_by_our_la_names(payload):
-    out = payload(_StubSession)
-    assert out["year"] == LATEST
-    assert out["secondary"]["attainment_8_by_la"] == {
-        "Kensington and Chelsea": 54.5,
-        "Bristol, City of": 46.3,
-    }
-
-
-def test_an_la_without_a_dfe_figure_is_absent(payload):
-    by_la = payload(_StubSession)["secondary"]["attainment_8_by_la"]
-    assert "City of London" not in by_la
-    assert "West Sussex" not in by_la
-    assert None not in by_la.values()
-
-
-def test_no_dfe_figures_for_the_year_give_an_empty_map(payload):
-    assert payload(_OldYearOnly) == {"year": LATEST, "secondary": {"attainment_8_by_la": {}}}
-
-
-def test_a_missing_mart_gives_an_empty_map(payload):
-    assert payload(_NoMart)["secondary"]["attainment_8_by_la"] == {}
-
-
-def test_the_endpoint_serves_dfe_figures(monkeypatch):
-    from backend import app as app_module
-    from backend import database as database_module
-
-    monkeypatch.setattr(app_module, "load_school_data", _df)
-    monkeypatch.setattr(database_module, "SessionLocal", _StubSession)
-    resp = TestClient(app_module.app).get("/api/la-averages")
-    assert resp.status_code == 200, resp.text
-    assert resp.json()["secondary"]["attainment_8_by_la"]["Kensington and Chelsea"] == 54.5

docs/superpowers/specs/2026-10-06-ks4-2023-24-and-la-averages-design.md

@@ -0,0 +1,300 @@
+# 2023/24 Results and DfE LA Averages — Design
+
+**Date:** 2026-10-06
+**Status:** approved design, not yet implemented
+**Scope:** `tap_uk_ees` (KS4 results, KS4 information, new LA stream),
+`safe_numeric`, new `fact_ks4_la_averages` mart, annual EES DAG selector,
+`/api/la-averages`, secondary search rows and map cards
+**Fixes:** audit findings C2 and H2, from the 3 Oct 2026 accuracy audit
+
+## Goal
+
+Show every 2023/24 result and school-information figure DfE published, and
+compare each state-funded secondary school with DfE's own local-authority
+average.
+
+## The problem
+
+### C2: 2023/24 is empty
+
+Bishop Stopford School (137086) shows Attainment 8 60.7 for 2022/23, nothing
+for 2023/24 and 58.7 for 2024/25. DfE's 2023/24 figures are Attainment 8 64.1,
+Progress 8 +1.02 and English and maths grade 4+ 91.7%. 2023/24 is the last year
+DfE published Progress 8 (2024/25 has no KS2 baseline), so the site shows no
+recent Progress 8 for any school. Site-wide, 4,170 listed schools have a DfE
+2023/24 Attainment 8 and 3,384 a Progress 8.
+
+There are three separate causes.
+
+1. **KS4 results.** The 2024/25 release's
+   `202425_performance_tables_schools_final.csv` is a time series: it holds
+   2022/23, 2023/24 and 2024/25 under the current column names, with the right
+   values (Bishop Stopford 2023/24: 64.1, 1.02, 91.7). The 2023/24 release's own
+   `202324_performance_tables_schools_final.csv`, re-issued on 10 March 2026,
+   uses the older names (`t_pupils`, `avg_att8`, `avg_p8score`,
+   `pt_l2basics_94` …). `EESDatasetStream` reads releases in the API's order,
+   newest first, so the old file is read last. Its rows carry none of the
+   declared fields, and target-postgres upserts on the stream's primary key
+   (`append_only = not key_properties` in meltanolabs-target-postgres 0.8.0),
+   so they overwrite the good 2023/24 rows with nulls.
+
+   The two files share the keys of every "Total" row, but 34,254 of the 57,090
+   2023/24 sub-group rows use different labels ("Low prior" against "Low prior
+   attainment"). Those old-label rows sit in `raw.ees_ks4_performance` with
+   null measures. Nothing reads them.
+
+2. **KS4 school information.** 2023/24 information exists only in the 2023/24
+   release, in `202324_information_about_schools_final.csv`, with the older
+   names. `EESKS4InfoStream` declares the newer ones, so prior attainment,
+   SEN percentages, disadvantage gaps and Progress 8 banding are null for
+   2023/24.
+
+3. **KS2 school information.** The 2023/24 file
+   `ks2_school_information_data.csv` uses the declared names, and pupil counts
+   load (school 147411: 818 pupils, 112 eligible). Its percentages are written
+   with a sign (`ptfsm6cla1a = "34%"`). `safe_numeric` accepts only
+   `^-?[0-9]+(\.[0-9]+)?$`, so disadvantaged, EAL, SEN and mobility percentages
+   are null for every school in 2023/24.
+
+DfE published no school-level KS2 information file for 2022/23 (the 2023/24
+release's attainment file carries 2022/23 attainment rows only). Those nulls are
+a gap in the source, not a defect.
+
+### H2: "vs LA avg" uses the wrong average
+
+`/api/la-averages` (`backend/app.py`) takes an unweighted mean of every school
+in the LA with an Attainment 8 score, independent and special schools included.
+Independent schools score low because DfE measures exclude IGCSEs, and special
+schools score low for other reasons, so the average is too low almost
+everywhere. Kensington and Chelsea's "LA avg" is 35.2: the mean of 6 state
+schools (54.9) and 8 independent schools (20.4). DfE's figure is 54.5. Of the
+151 LAs the audit compared, ours was lower in 147, by 7.1 points on average and
+by up to 19.3, so most secondary schools look better than their area.
+
+DfE's LA averages are already in the file the pipeline downloads for the
+England averages: the "summary, all state-funded" data set
+(`data-catalogue/data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv`). Its
+`Local authority` / `All state-funded` / Total rows match DfE's published
+performance-table LA averages (RECTYPE 4) for all 152 LAs in 2024/25, with no
+difference. `EESKs4NationalStream` keeps the England row and discards them.
+"All state-funded" is the same population as the England benchmark the site
+already shows.
+
+Computing the average ourselves from state-funded schools, weighted by pupils,
+was tested and rejected: it differs from DfE's figure by 1.1 points on average
+and is never exact.
+
+## Non-goals
+
+- 2022/23 KS2 school information (DfE published none).
+- An "excludes IGCSEs" note wherever an independent school's Attainment 8
+  appears. This change only stops comparing independent schools with the LA.
+- Showing 2023/24 Progress 8 in the GCSE section's headline. The history section
+  shows it once the data loads; the 2024/25 banner stays true.
+- Updating the fixed EES data-set id when DfE publishes 2025/26. It already
+  feeds the England averages; the LA stream shares it.
+- LA comparisons for other measures or on other pages.
+- Deleting the leftover old-label raw rows automatically.
+
+## The rules
+
+### The newest release owns every year it contains (KS4 results only)
+
+`EESDatasetStream` gets an opt-in class attribute,
+`_newest_release_owns_period: bool = False`. `EESKS4PerformanceStream` sets it
+to `True`. When it is on:
+
+- releases are processed newest first by `time_period` (from the release slug),
+  not in the API's order;
+- the stream records each `time_period` it has emitted;
+- in each older release, rows whose `time_period` a newer release already
+  emitted are dropped, and the stream logs how many it skipped and for which
+  years;
+- years only an older release contains are emitted as before.
+
+The filter is a pure function, testable without a download. Other streams keep
+the current behaviour. A general rule would be wrong: the 2024/25 KS2 file holds
+98,448 of the 955,956 rows the 2023/24 release has for 2023/24.
+
+### KS4 information: old names
+
+`EESKS4InfoStream._column_renames` maps the 2023/24 names onto the declared
+fields:
+
+| 2023/24 column | Declared field |
+|---|---|
+| `t_allks_pupils` | `allks_pupil_count` |
+| `t_allks_boys` | `allks_boys_count` |
+| `t_allks_girls` | `allks_girls_count` |
+| `t_pupils` | `endks4_pupil_count` |
+| `avg_ks2_scaledscore` | `ks2_scaledscore_average` |
+| `pt_sen_with_ehcp` | `sen_with_ehcp_pupil_percent` |
+| `pt_sen` | `sen_pupil_percent` |
+| `pt_sen_no_ehcp` | `sen_no_ehcp_pupil_percent` |
+| `diffn_att8` | `attainment8_diffn` |
+| `diffn_p8mea` | `progress8_diffn` |
+| `p8_banding` | `progress8_banding` |
+
+Newer files contain none of the old names, so they are unaffected.
+
+### `safe_numeric` accepts a trailing `%`
+
+The pattern becomes `^-?[0-9]+(\.[0-9]+)?%?$` and the cast reads
+`rtrim(col, '%')`. A value such as `34%` can only mean 34. Suppression codes
+(`c`, `z`, `x` …) still become null.
+
+### LA averages
+
+A new stream, `ees_ks4_la`, reads the same CSV as `ees_ks4_national` and keeps
+rows where `geographic_level = 'Local authority'`,
+`establishment_type_group = 'All state-funded'`, `breakdown_topic = 'Total'` and
+`breakdown = 'Total'` (case-insensitive, as the national stream compares). It
+emits `time_period`, `old_la_code`, `new_la_code`, `la_name` and the 8 headline
+measures the national stream emits (`_KS4_NATIONAL_COL_MAP`). Primary key:
+(`time_period`, `old_la_code`). The two streams share one download-and-filter
+helper.
+
+`old_la_code` is the GIAS LA code (`local_authority_code`), so schools join on
+the code, not the name. Names match today for every LA DfE publishes; DfE
+publishes no figure for City of London.
+
+### Which schools get a gap
+
+The search row and the map card show "vs LA avg" only when the school has an
+Attainment 8 score, is neither special (`isSpecialSchool`) nor independent
+(`isIndependentSchool`: "independent" in the GIAS type, which covers "Other
+independent school" and "Other independent special school"), and its LA has a
+DfE figure for the year the endpoint serves.
+
+## Delivery
+
+Two PRs, as for C1.
+
+### PR 1: pipeline
+
+- `pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py`: release
+  precedence, KS4 information renames, `ees_ks4_la` stream registered in
+  `discover_streams`, shared national/LA helper.
+- `pipeline/transform/macros/safe_numeric.sql`: trailing `%`.
+- `pipeline/transform/models/staging/`: source `raw.ees_ks4_la` and
+  `stg_ees_ks4_la` (view): `cast(old_la_code as integer) as la_code`,
+  `cast(time_period as integer) as year`, `la_name`, measures via
+  `safe_numeric`, named as in `stg_ees_ks4_national`.
+- `pipeline/transform/models/marts/fact_ks4_la_averages.sql` (table): one row
+  per (`year`, `la_code`) with `la_name` and the columns of
+  `fact_ks4_national_averages`. Schema tests: unique (`year`, `la_code`);
+  `year`, `la_code` and `la_name` not null.
+- Data tests in `pipeline/transform/tests/`:
+  - `assert_ks4_years_have_results`: every year in `stg_ees_ks4` has a non-null
+    Attainment 8 for at least 50% of its rows. DfE's files reach 82% each year;
+    2023/24 loads at 0% today. Pre-2019 years come from the legacy model and are
+    not tested.
+  - `assert_ks2_info_percentages_loaded`: for every year in `stg_ees_ks2` where
+    at least 1,000 rows have `total_pupils`, at least 90% of those rows have
+    `disadvantaged_pct`. DfE's files reach 96–97%. 2022/23 has no pupil counts
+    and is skipped.
+  - `assert_ks4_la_averages_cover_las`: the latest year in
+    `fact_ks4_la_averages` has at least 145 LAs (DfE: 152).
+- `pipeline/dags/school_data_pipeline.py`: the annual EES build selects
+  `stg_ees_ks4_la+`.
+- `docs/ARCHITECTURE.md`: LA averages come from DfE's data set.
+
+### PR 2: backend and UI (after the EES DAG has run on PR 1)
+
+- `backend/models.py`: `Ks4LaAverage` for `marts.fact_ks4_la_averages`.
+- `backend/app.py` `/api/la-averages`: the year is the latest with any school
+  Attainment 8 (as now). It reads that year's rows from the mart and keys each
+  `attainment_8_score` by our LA name, through the `local_authority_code` →
+  `local_authority` pairs in the school data. The response shape is unchanged.
+  No rows for that year, a missing table or a query error give an empty map,
+  logged, never another year's figures and never a computed mean.
+- `nextjs-app/lib/utils.ts`: `isIndependentSchool(school)`.
+- `nextjs-app/components/SecondarySchoolRow.tsx` and
+  `nextjs-app/components/LeafletMapInner.tsx`: the rule in "Which schools get a
+  gap".
+- `e2e/tests/journeys.spec.ts`: the two journeys under Testing.
+
+## Testing
+
+**Extractor (pytest, `pipeline/tests/`, new):**
+
+- precedence: with the flag on, a year in a newer release is emitted once, from
+  the newer release; a year only an older release has is kept; with the flag
+  off, every row passes;
+- releases arriving oldest first are still processed newest first;
+- KS4 information: an old-format row yields `endks4_pupil_count`,
+  `ks2_scaledscore_average`, `progress8_banding` and the rest of the table;
+- LA filter: a small CSV with national, regional, LA and sub-group rows yields
+  one row per LA and year, with the declared fields.
+
+**dbt (local `pgserver`, as for C1):**
+
+- unit test on `stg_ees_ks2`: `34%` → 34, `34` → 34, `c` → null;
+- unit test on `stg_ees_ks4_la` or the mart: codes and years cast, measures
+  carried;
+- the three data tests and the schema tests above;
+- `pipeline/tests/test_dag_selectors.py` passes with the new selector.
+
+**Backend (pytest):**
+
+- the response gives the mart's figure where the fixture's plain mean differs;
+- matching works by code when the mart's `la_name` differs from ours;
+- a mart without the served year, and a missing table, give an empty map.
+
+**Front end (Jest):**
+
+- `isIndependentSchool` for "Other independent school", "Other independent
+  special school", "Academy converter" and null;
+- `SecondarySchoolRow` and the map card show no gap for an independent school
+  and show one for a state school with an LA figure.
+
+**E2E (`journeys.spec.ts`, PR 2):**
+
+- C2: Bishop Stopford (137086) history shows 2023/24 Attainment 8 64.1 and
+  Progress 8 +1.02. These are final figures, so the journey stays stable.
+- H2: a search returning an independent and a state secondary: the independent
+  row has no "vs LA avg", the state row has one. The exact gap is not asserted:
+  DfE's 2025/26 provisional KS4 data is due and would change it.
+
+## Rollout and verification
+
+1. PR 1 merges to staging. Tudor runs `school_data_annual_ees` on staging.
+2. Through the staging API: Bishop Stopford 2023/24 Attainment 8 64.1 and
+   Progress 8 1.02; school 147411 2023/24 disadvantaged 34; the LA mart has 152
+   LAs for 2024/25. If a data test fails, the build stops before search sync;
+   investigate before going further.
+3. PR 2 merges, so the post-merge E2E gate runs against loaded data.
+4. Production, Tudor's decision, in order: promote PR 1, run the EES DAG on
+   production, promote PR 2. If PR 2 arrives first, the endpoint returns an
+   empty map and rows show no gap, never a wrong one.
+5. Optional cleanup, in the PR 1 description for Tudor:
+   `delete from raw.ees_ks4_performance where time_period = '202324' and pupil_count is null`.
+   New-format rows always have `pupil_count` (suppressed values are `c` or `z`,
+   not null), so this removes exactly the old-label leftovers.
+6. Validation on production against DfE's files: every listed school's 2023/24
+   Attainment 8, Progress 8 and English and maths figures match; the 2023/24 KS4
+   and KS2 information fields match; `/api/la-averages` equals DfE for all 152
+   LAs; independent rows show no gap. Then mark C2 and H2 resolved in the audit
+   report.
+
+## Expected visible change
+
+- Secondary history charts and tables run unbroken from 2022/23 to 2024/25, and
+  2023/24 Progress 8 appears for about 3,400 schools.
+- 2023/24 school information (KS2 and KS4) fills in.
+- "vs LA avg" drops for most state secondaries. In Kensington and Chelsea, a
+  school with Attainment 8 60.0 moves from +24.8 to +5.5.
+- Independent schools and City of London schools show no LA gap.
+
+## Risks
+
+- **Data-test thresholds stop a good build.** They were set from DfE's own
+  files (82% and 96–97% against 50% and 90%). A failure means the data changed
+  shape, which is what they are for.
+- **`safe_numeric` is shared by 14 models.** Values written as `n%` were null
+  and become numbers. No DfE column uses `%` for anything but a percentage.
+  The staging EES run is the check.
+- **Precedence drops data an older release holds more completely.** It is
+  opt-in for KS4 results, where both files hold the same 57,090 2023/24 keys.
+- **The fixed EES data-set id goes stale** when 2025/26 is published. The
+  endpoint's year check then gives no gap rather than a mismatched one.

e2e/tests/journeys.spec.ts

@@ -417,15 +417,12 @@ test('a secondary search row compares its Attainment 8 with the LA average', asy
   // guards the comparison itself; the unit test pins the cache mode.
   const la = await (await page.request.get('/api/la-averages')).json();
   const averages: Record<string, number> = la.secondary?.attainment_8_by_la ?? {};
-  // DfE publishes about 152 LA averages. An empty map (a missing mart, or a
-  // year the LA data set has not reached) hides every comparison: fail, not skip.
-  expect(Object.keys(averages).length).toBeGreaterThan(100);
   const res = await page.request.get('/api/schools?search=school&phase=secondary&page_size=50');
   expect(res.ok()).toBeTruthy();
   const school = ((await res.json()).schools ?? []).find(
     (s: { attainment_8_score?: number | null; local_authority?: string; school_type?: string }) =>
       s.attainment_8_score != null && s.local_authority != null && averages[s.local_authority] != null
-      && !/special|pupil referral|alternative provision|independent/i.test(s.school_type ?? ''));
+      && !/special|pupil referral|alternative provision/i.test(s.school_type ?? ''));
   test.skip(!school, 'no mainstream secondary with an LA average here');
 
   await searchByName(page, school.school_name);
@@ -436,52 +433,6 @@ test('a secondary search row compares its Attainment 8 with the LA average', asy
   await expect(stats.getByText(/vs LA avg/)).toBeVisible();
 });
 
-test('an independent secondary shows its Attainment 8 without an LA comparison', async ({ page }) => {
-  // DfE's LA averages cover state-funded schools, and an independent school's
-  // Attainment 8 leaves out IGCSEs, so a gap would mislead (audit H2). A state
-  // school on the same page must show its gap first, so the absence is real.
-  const la = await (await page.request.get('/api/la-averages')).json();
-  const averages: Record<string, number> = la.secondary?.attainment_8_by_la ?? {};
-  expect(Object.keys(averages).length).toBeGreaterThan(100);
-
-  type Row = { urn: number; school_type?: string; local_authority?: string; attainment_8_score?: number | null };
-  const compared = (s: Row) => s.attainment_8_score != null && s.local_authority != null
-    && averages[s.local_authority] != null
-    && !/special|pupil referral|alternative provision/i.test(s.school_type ?? '');
-  let found: { la: string; state: Row; independent: Row } | null = null;
-  for (const name of ['Kensington and Chelsea', 'Westminster', 'Camden', 'Hammersmith and Fulham', 'Barnet']) {
-    // The search page asks for the same first 50 schools.
-    const res = await page.request.get(`/api/schools?search=${encodeURIComponent(name)}&phase=secondary&page_size=50`);
-    const schools: Row[] = ((await res.json()).schools ?? []).filter(compared);
-    const independent = schools.find(s => /independent/i.test(s.school_type ?? ''));
-    const state = schools.find(s => !/independent/i.test(s.school_type ?? ''));
-    if (independent && state) { found = { la: name, state, independent }; break; }
-  }
-  test.skip(!found, 'no LA here lists a state and an independent secondary on one page');
-
-  await page.goto(`/?search=${encodeURIComponent(found!.la)}&phase=secondary`);
-  const stats = (urn: number) => page.locator(`a[href^="/school/${urn}-"]`).first()
-    .locator('xpath=ancestor::div[contains(@class, "__rowContent")][1]')
-    .locator('[class*="__line3"]');
-  await expect(stats(found!.state.urn).getByText(/vs LA avg/)).toBeVisible({ timeout: 15_000 });
-  const independent = stats(found!.independent.urn);
-  await expect(independent.getByText(found!.independent.attainment_8_score!.toFixed(1))).toBeVisible();
-  await expect(independent.getByText(/vs LA avg/)).toHaveCount(0);
-});
-
-test('a secondary shows its 2023/24 GCSE results, the last year DfE published Progress 8', async ({ page }) => {
-  // DfE re-issued its 2023/24 file under older column names, and every
-  // school's 2023/24 row loaded empty (audit C2). These are DfE's final
-  // figures for Bishop Stopford School, so they do not change.
-  await page.goto('/school/137086');
-  const history = page.locator('#history');
-  await history.getByText('View raw year-by-year data').click();
-  const row = history.getByRole('row', { name: /2023\/24/ });
-  await expect(row).toContainText('64.1');
-  await expect(row).toContainText('+1.0');
-  await expect(row).toContainText('91.7%');
-});
-
 test('a phase outside primary/secondary filters to that phase, not to everything', async ({ page }) => {
   // The search page offers every GIAS phase, but the API only knew the grouped
   // ones and silently dropped the rest — so "Nursery" returned primaries.

nextjs-app/__tests__/components/LeafletMapInner.test.tsx

@@ -105,20 +105,3 @@ it('puts the card back when the pins are rebuilt for a reason other than the sch
   expect(container.querySelector('.sc-popup')).toHaveTextContent('Southmead Primary School');
   expect(container.querySelectorAll('.sc-pin--selected')).toHaveLength(1);
 });
-
-it('compares a state secondary with its LA average on the card, but not an independent one', () => {
-  const state: School = { ...base, urn: 3, school_name: 'Holland Park School', school_type: 'Academy converter',
-    phase: 'Secondary', local_authority: 'Kensington and Chelsea', attainment_8_score: 60,
-    latitude: 51.5, longitude: -0.2, distance: 0.3 };
-  const independent: School = { ...state, urn: 4, school_name: 'Abbey Gate College',
-    school_type: 'Other independent school', attainment_8_score: 20.4 };
-  const laAverages = { 'Kensington and Chelsea': 54.5 };
-
-  const { container, rerender } = renderMap({ schools: [state, independent], laAverages, selectedUrn: 3 });
-  expect(container.querySelector('.sc-popup')).toHaveTextContent('60.0 Att 8 +5.5 vs LA');
-
-  rerender({ schools: [state, independent], laAverages, selectedUrn: 4 });
-  const card = container.querySelector('.sc-popup')!;
-  expect(card).toHaveTextContent('20.4 Att 8');
-  expect(card).not.toHaveTextContent(/vs LA/);
-});

nextjs-app/__tests__/components/SecondarySchoolRow.test.tsx

@@ -116,38 +116,3 @@ describe('SecondarySchoolRow shares the school page flags', () => {
     expect(screen.getByText('Fee-paying')).toBeInTheDocument();
   });
 });
-
-describe('SecondarySchoolRow LA comparison', () => {
-  it('compares a state school with its LA average', () => {
-    render(
-      <SecondarySchoolRow
-        school={{ ...base, school_type: 'Academy converter', attainment_8_score: 60 }}
-        laAvgAttainment8={54.5}
-      />,
-    );
-    expect(screen.getByText(/\+5\.5 vs LA avg/)).toBeInTheDocument();
-  });
-
-  it('keeps the comparison for a school whose type is unknown', () => {
-    render(
-      <SecondarySchoolRow
-        school={{ ...base, school_type: null, attainment_8_score: 60 } as unknown as School}
-        laAvgAttainment8={54.5}
-      />,
-    );
-    expect(screen.getByText(/\+5\.5 vs LA avg/)).toBeInTheDocument();
-  });
-
-  it("shows an independent school's Attainment 8 without an LA comparison", () => {
-    // DfE's LA average covers state-funded schools; an independent's
-    // Attainment 8 leaves out IGCSEs (audit H2).
-    render(
-      <SecondarySchoolRow
-        school={{ ...base, school_type: 'Other independent school', attainment_8_score: 20.4 }}
-        laAvgAttainment8={54.5}
-      />,
-    );
-    expect(screen.getByText('20.4')).toBeInTheDocument();
-    expect(screen.queryByText(/vs LA avg/)).not.toBeInTheDocument();
-  });
-});

nextjs-app/__tests__/lib/utils.test.ts

@@ -391,20 +391,3 @@ describe('singleSexLabel', () => {
     expect(singleSexLabel(undefined)).toBeNull();
   });
 });
-
-describe('isIndependentSchool', () => {
-  const { isIndependentSchool } = require('@/lib/utils');
-
-  it('matches both GIAS independent types', () => {
-    expect(isIndependentSchool({ school_type: 'Other independent school' })).toBe(true);
-    expect(isIndependentSchool({ school_type: 'Other independent special school' })).toBe(true);
-  });
-
-  it('does not match state-funded types or a missing type', () => {
-    for (const t of ['Academy converter', 'Community school', 'Free schools', 'Non-maintained special school']) {
-      expect(isIndependentSchool({ school_type: t })).toBe(false);
-    }
-    expect(isIndependentSchool({ school_type: null })).toBe(false);
-    expect(isIndependentSchool({})).toBe(false);
-  });
-});

nextjs-app/components/LeafletMapInner.tsx

@@ -9,7 +9,7 @@ import { useEffect, useRef, useState } from 'react';
 import L from 'leaflet';
 import 'leaflet/dist/leaflet.css';
 import type { School } from '@/lib/types';
-import { schoolUrl, isSpecialSchool, isIndependentSchool, buildOfstedListBadge, listRwmValue } from '@/lib/utils';
+import { schoolUrl, isSpecialSchool, buildOfstedListBadge, listRwmValue } from '@/lib/utils';
 
 interface LeafletMapInnerProps {
   schools: School[];
@@ -70,7 +70,7 @@ function metricHtml(school: School, { nationalAvgRwm, laAverages }: CardContext)
     const score = school.attainment_8_score;
     const laAvg = school.local_authority ? (laAverages?.[school.local_authority] ?? null) : null;
     let delta = '';
-    if (!special && !isIndependentSchool(school) && laAvg != null) {
+    if (!special && laAvg != null) {
       const diff = Math.round((score - laAvg) * 10) / 10;
       // Att8 runs 0–90 in 0.1 steps; ±0.5 is meaningful, where RWM needs ±2.
       const cls = diff >= 0.5 ? 'sc-up' : diff <= -0.5 ? 'sc-down' : '';

nextjs-app/components/SecondarySchoolRow.tsx

@@ -11,7 +11,7 @@
 'use client';
 
 import type { School } from '@/lib/types';
-import { buildOfstedListBadge, getPhaseStyle, schoolUrl, formatAgeRange, isProposedToClose, isSpecialSchool, isIndependentSchool } from '@/lib/utils';
+import { buildOfstedListBadge, getPhaseStyle, schoolUrl, formatAgeRange, isProposedToClose, isSpecialSchool } from '@/lib/utils';
 import { schoolFlags, schoolTypeLabel } from '@/lib/schoolFacts';
 import styles from './SecondarySchoolRow.module.css';
 
@@ -45,12 +45,10 @@ export function SecondarySchoolRow({
   const att8 = school.attainment_8_score;
   // The school's own Attainment 8 is a same-school figure — shown whenever it
   // exists (special schools included; their type tag on line 2 gives context).
-  // Only the vs-LA-average delta, a benchmark comparison, is dropped: for
-  // special schools / PRUs / AP, whose pupils aren't measured against it
-  // fairly, and for independent schools, because DfE's LA average covers
-  // state-funded schools and an independent's Attainment 8 leaves out IGCSEs.
+  // Only the vs-LA-average delta, a benchmark comparison, is dropped for
+  // special schools / PRUs / AP, whose pupils aren't measured against it fairly.
   const laDelta =
-    att8 != null && !isSpecialSchool(school) && !isIndependentSchool(school) && laAvgAttainment8 != null
+    att8 != null && !isSpecialSchool(school) && laAvgAttainment8 != null
       ? att8 - laAvgAttainment8
       : null;
 

nextjs-app/lib/utils.ts

@@ -758,16 +758,6 @@ export function isSpecialSchool(school: { school_type?: string | null }): boolea
   return /\bspecial\b/.test(t) || /pupil referral/.test(t) || /alternative provision/.test(t);
 }
 
-/**
- * Independent (fee-paying) schools: GIAS types "Other independent school" and
- * "Other independent special school". DfE's Attainment 8 for them leaves out
- * IGCSEs, and DfE's LA averages cover state-funded schools only, so callers
- * drop the "vs LA avg" comparison for them (audit H2).
- */
-export function isIndependentSchool(school: { school_type?: string | null }): boolean {
-  return /\bindependent\b/i.test(school.school_type ?? '');
-}
-
 /**
  * Whether GIAS records a religious character. "None", "Does not apply" and
  * "Not applicable" are the register's ways of saying it has none; the place