docs: name the benchmarks computed from our dataset (review)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/ARCHITECTURE.md

@@ -61,6 +61,12 @@ history from the full DataFrame and supplementary data from marts. Comparisons
 batch supplementary queries across selected URNs. Async routes still contain
 synchronous dependency calls; a fully asynchronous database layer is not present.
 
+DfE's official benchmarks live in marts: `fact_ks2_national_averages` and
+`fact_ks4_national_averages` for England, and `fact_ks4_la_averages` (all
+state-funded schools, per LA) for the search rows' "vs LA avg".
+`data_loader.compute_benchmarks` computes further state-school benchmarks from
+our own dataset; they are not DfE figures and are labelled as computed.
+
 ## Frontend boundaries
 
 `app/(frontend)` owns the public root layout and pages. `app/(payload)` owns the

pipeline/dags/school_data_pipeline.py

@@ -193,7 +193,7 @@ with DAG(
 
     dbt_build_ees = BashOperator(
         task_id="dbt_build",
-        bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ees_ks2+ stg_legacy_ks2+ stg_ees_ks4+ stg_legacy_ks4+ stg_ees_census+ stg_ees_admissions+ stg_ees_ks2_national+ stg_ees_ks4_national+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+ stg_ees_ks4_destinations_national+ stg_ees_ks5_destinations_national+",
+        bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ees_ks2+ stg_legacy_ks2+ stg_ees_ks4+ stg_legacy_ks4+ stg_ees_census+ stg_ees_admissions+ stg_ees_ks2_national+ stg_ees_ks4_national+ stg_ees_ks4_la+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+ stg_ees_ks4_destinations_national+ stg_ees_ks5_destinations_national+",
     )
 
     sync_typesense_ees = BashOperator(

pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/ks4_info.py

@@ -0,0 +1,47 @@
+"""KS4 school information: the fields the stream declares, and the older
+names DfE used for them.
+
+2023/24 information exists only in the 2023/24 release, whose
+202324_information_about_schools_final.csv (re-issued 10 March 2026) uses the
+older names. Without the renames every 2023/24 field loaded as null (audit C2).
+Newer files contain none of the older names, so the renames leave them alone.
+
+Free of the Singer SDK so CI's pytest can load it.
+"""
+
+# Declared Singer fields besides the required time_period and school_urn.
+KS4_INFO_FIELDS = (
+    "school_laestab",
+    "school_name",
+    "establishment_type_group",
+    "reldenom",
+    "admpol_pt",
+    "egender",
+    "agerange",
+    "allks_pupil_count",
+    "allks_boys_count",
+    "allks_girls_count",
+    "endks4_pupil_count",
+    "ks2_scaledscore_average",
+    "sen_with_ehcp_pupil_percent",
+    "sen_pupil_percent",
+    "sen_no_ehcp_pupil_percent",
+    "attainment8_diffn",
+    "progress8_diffn",
+    "progress8_banding",
+)
+
+# 2023/24 column name → declared field.
+KS4_INFO_RENAMES = {
+    "t_allks_pupils": "allks_pupil_count",
+    "t_allks_boys": "allks_boys_count",
+    "t_allks_girls": "allks_girls_count",
+    "t_pupils": "endks4_pupil_count",
+    "avg_ks2_scaledscore": "ks2_scaledscore_average",
+    "pt_sen_with_ehcp": "sen_with_ehcp_pupil_percent",
+    "pt_sen": "sen_pupil_percent",
+    "pt_sen_no_ehcp": "sen_no_ehcp_pupil_percent",
+    "diffn_att8": "attainment8_diffn",
+    "diffn_p8mea": "progress8_diffn",
+    "p8_banding": "progress8_banding",
+}

pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/ks4_summary.py

@@ -0,0 +1,56 @@
+"""DfE's KS4 "summary, all state-funded" data set (Key stage 4 performance).
+
+One CSV holds England, regional and local-authority headline rows for every
+year since 2018/19. The England stream (ees_ks4_national) and the LA stream
+(ees_ks4_la) both read it. Its LA rows match DfE's published performance-table
+LA averages exactly (audit H2). Suppressed values ('z', 'x') become NULL in
+dbt; Progress 8 is 'z' in years with no KS2 baseline (2024/25): DfE policy,
+not missing data.
+
+Free of the Singer SDK so CI's pytest can load it.
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+KS4_SUMMARY_CSV_URL = (
+    "https://explore-education-statistics.service.gov.uk/data-catalogue/"
+    "data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv"
+)
+
+# CSV column → Singer field: the same 8 headline measures at every level.
+KS4_HEADLINE_COL_MAP = {
+    "attainment8_average":    "attainment_8_score",
+    "progress8_average":      "progress_8_score",
+    "engmath_94_percent":     "english_maths_standard_pass_pct",
+    "engmath_95_percent":     "english_maths_strong_pass_pct",
+    "ebacc_entering_percent": "ebacc_entry_pct",
+    "ebacc_94_percent":       "ebacc_standard_pass_pct",
+    "ebacc_95_percent":       "ebacc_strong_pass_pct",
+    "ebacc_aps_average":      "ebacc_avg_score",
+}
+
+
+def headline_rows(df: pd.DataFrame, geographic_level: str) -> pd.DataFrame:
+    """All-pupil rows for all state-funded schools at one geographic level
+    ("National" or "Local authority"). Column names are lower-cased first;
+    a filter column the file lacks is not applied."""
+    df = df.copy()
+    df.columns = [c.strip().lower() for c in df.columns]
+    for col, want in (
+        ("geographic_level", geographic_level),
+        ("establishment_type_group", "All state-funded"),
+        ("breakdown_topic", "Total"),
+        ("breakdown", "Total"),
+    ):
+        if col in df.columns:
+            df = df[df[col].str.strip().str.lower() == want.lower()]
+    return df
+
+
+def headline_record(row: pd.Series, keys: tuple[str, ...]) -> dict[str, str]:
+    """A Singer record: the identifying columns, then the headline measures."""
+    record = {key: str(row.get(key, "")).strip() for key in keys}
+    for csv_col, field in KS4_HEADLINE_COL_MAP.items():
+        record[field] = str(row.get(csv_col, "")).strip()
+    return record

pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/release_precedence.py

@@ -0,0 +1,59 @@
+"""Which release a row comes from when DfE re-publishes a year.
+
+DfE re-publishes earlier years inside later releases: the 2024/25 KS4 results
+file holds 2022/23, 2023/24 and 2024/25. The 2023/24 release's own file,
+re-issued on 10 March 2026 under older column names, was read after it and
+overwrote every 2023/24 row with blanks (audit C2). A stream that opts in
+treats the newest release as the authority for every year it contains.
+
+Free of the Singer SDK so CI's pytest, which installs only the backend's
+requirements, can load it.
+"""
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+_SLUG_YEAR = re.compile(r"^(\d{4})-(\d{2})(?:-|$)")
+
+
+def slug_to_time_period(slug: str) -> str | None:
+    """A release slug's academic year as a time_period: '2022-23' → '202223'.
+
+    Suffixed slugs ('2024-25-revised', '2025-26-provisional') give the same
+    year, so newest_first places them by year; read as unknown, a revised
+    release went last and lost its year to the first release.
+    """
+    match = _SLUG_YEAR.match(slug or "")
+    return match.group(1) + match.group(2) if match else None
+
+
+def newest_first(releases: list[dict]) -> list[dict]:
+    """Releases by time_period, newest first. A release whose time_period is
+    unknown goes last, in the order given."""
+    dated = [r for r in releases if r.get("time_period")]
+    undated = [r for r in releases if not r.get("time_period")]
+    return sorted(dated, key=lambda r: r["time_period"], reverse=True) + undated
+
+
+def periods_in(df: pd.DataFrame) -> set[str]:
+    """The years a release's rows cover."""
+    if "time_period" not in df.columns:
+        return set()
+    return set(df["time_period"].astype(str).str.strip()) - {""}
+
+
+def drop_owned_periods(
+    df: pd.DataFrame, owned: set[str]
+) -> tuple[pd.DataFrame, dict[str, int]]:
+    """Drop the rows for years a newer release already supplied.
+
+    Returns the rows kept and, for each year dropped, how many rows went.
+    """
+    if "time_period" not in df.columns or not owned:
+        return df, {}
+    periods = df["time_period"].astype(str).str.strip()
+    dropped = periods.isin(owned)
+    skipped = {str(k): int(v) for k, v in periods[dropped].value_counts().items()}
+    return df[~dropped], skipped

pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py

@@ -17,6 +17,20 @@ import requests
 from singer_sdk import Stream, Tap
 from singer_sdk import typing as th
 
+from tap_uk_ees.ks4_info import KS4_INFO_FIELDS, KS4_INFO_RENAMES
+from tap_uk_ees.ks4_summary import (
+    KS4_HEADLINE_COL_MAP,
+    KS4_SUMMARY_CSV_URL,
+    headline_record,
+    headline_rows,
+)
+from tap_uk_ees.release_precedence import (
+    drop_owned_periods,
+    newest_first,
+    periods_in,
+    slug_to_time_period,
+)
+
 CONTENT_API_BASE = (
     "https://content.explore-education-statistics.service.gov.uk/api"
 )
@@ -31,14 +45,6 @@ def get_content_release_id(publication_slug: str) -> str:
     return resp.json()["id"]
 
 
-def _slug_to_time_period(slug: str) -> str | None:
-    """Convert a release slug like '2022-23' to a time_period like '202223'."""
-    parts = slug.split("-")
-    if len(parts) == 2 and len(parts[0]) == 4 and len(parts[1]) == 2:
-        return parts[0] + parts[1]
-    return None
-
-
 def get_all_releases(publication_slug: str) -> list[dict]:
     """Return all releases for a publication as dicts with 'id' and 'time_period'.
 
@@ -63,7 +69,7 @@ def get_all_releases(publication_slug: str) -> list[dict]:
             total_pages = paging.get("totalPages", 1)
 
         for r in releases:
-            time_period = _slug_to_time_period(r.get("slug", ""))
+            time_period = slug_to_time_period(r.get("slug", ""))
             result.append({"id": r["id"], "time_period": time_period})
 
         if page >= total_pages:
@@ -88,6 +94,8 @@ class EESDatasetStream(Stream):
     target CSV path inside the ZIP (substring match, not exact).
     Subclasses may set _column_renames to map messy CSV column names to
     clean Singer field names before yielding records.
+    Subclasses may set _newest_release_owns_period when DfE re-publishes
+    earlier years in later releases and the newest copy is the authority.
     """
 
     replication_key = None
@@ -96,6 +104,7 @@ class EESDatasetStream(Stream):
     _urn_column: str = "school_urn"  # column name for URN in the CSV
     _encoding: str = "utf-8"  # CSV file encoding (some DfE files use latin-1)
     _column_renames: dict = {}  # CSV column name → Singer field name
+    _newest_release_owns_period: bool = False  # see release_precedence.py
 
     def get_records(self, context):
         import pandas as pd
@@ -110,6 +119,9 @@ class EESDatasetStream(Stream):
         self.logger.info(
             "Found %d release(s) for %s", len(releases), self._publication_slug
         )
+        if self._newest_release_owns_period:
+            releases = newest_first(releases)
+        owned_periods: set[str] = set()
 
         for release in releases:
             release_id = release["id"]
@@ -163,6 +175,15 @@ class EESDatasetStream(Stream):
             if urn_col in df.columns:
                 df = df[df[urn_col].notna() & (df[urn_col] != "")]
 
+            if self._newest_release_owns_period:
+                df, skipped = drop_owned_periods(df, owned_periods)
+                for period, count in sorted(skipped.items()):
+                    self.logger.info(
+                        "Skipping %d rows for %s from release %s: a newer release supplied that year",
+                        count, period, release_id,
+                    )
+                owned_periods |= periods_in(df)
+
             self.logger.info("Emitting %d school-level rows from release %s", len(df), release_id)
 
             for _, row in df.iterrows():
@@ -251,6 +272,9 @@ class EESKS4PerformanceStream(EESDatasetStream):
     primary_keys = ["school_urn", "time_period", "breakdown_topic", "breakdown", "sex"]
     _publication_slug = "key-stage-4-performance"
     _target_filename = "performance_tables_schools"
+    # DfE's 2024/25 file re-publishes 2022/23 and 2023/24 under current names;
+    # the 2023/24 release's own file uses older ones (audit C2).
+    _newest_release_owns_period = True
     schema = th.PropertiesList(
         th.Property("time_period", th.StringType, required=True),
         th.Property("school_urn", th.StringType, required=True),
@@ -310,34 +334,20 @@ class EESKS4PerformanceStream(EESDatasetStream):
 
 
 # ── KS4 Information (wide format: one row per school, context/demographics) ──
-# File: 202425_information_about_schools_provisional.csv (38 cols)
+# Files: 202425_information_about_schools_final.csv (38 cols, current names);
+# 202324_information_about_schools_final.csv (60 cols, older names — the only
+# source of 2023/24 information). Field list and renames: ks4_info.py.
 
 class EESKS4InfoStream(EESDatasetStream):
     name = "ees_ks4_info"
     primary_keys = ["school_urn", "time_period"]
     _publication_slug = "key-stage-4-performance"
     _target_filename = "information_about_schools"
+    _column_renames = KS4_INFO_RENAMES
     schema = th.PropertiesList(
         th.Property("time_period", th.StringType, required=True),
         th.Property("school_urn", th.StringType, required=True),
-        th.Property("school_laestab", th.StringType),
-        th.Property("school_name", th.StringType),
-        th.Property("establishment_type_group", th.StringType),
-        th.Property("reldenom", th.StringType),
-        th.Property("admpol_pt", th.StringType),
-        th.Property("egender", th.StringType),
-        th.Property("agerange", th.StringType),
-        th.Property("allks_pupil_count", th.StringType),
-        th.Property("allks_boys_count", th.StringType),
-        th.Property("allks_girls_count", th.StringType),
-        th.Property("endks4_pupil_count", th.StringType),
-        th.Property("ks2_scaledscore_average", th.StringType),
-        th.Property("sen_with_ehcp_pupil_percent", th.StringType),
-        th.Property("sen_pupil_percent", th.StringType),
-        th.Property("sen_no_ehcp_pupil_percent", th.StringType),
-        th.Property("attainment8_diffn", th.StringType),
-        th.Property("progress8_diffn", th.StringType),
-        th.Property("progress8_banding", th.StringType),
+        *[th.Property(field, th.StringType) for field in KS4_INFO_FIELDS],
     ).to_dict()
 
 
@@ -564,37 +574,23 @@ class EESKs2NationalStream(Stream):
             yield record
 
 
-# ── KS4 National Headlines (national level only — one row per year) ──────────
-# Dataset: "National characteristics summary data" (Key stage 4 performance).
-# Official England state-funded headline measures, 2018/19 → latest.
-# Suppressed values ('z', 'x') → NULL downstream. Progress 8 is legitimately
-# absent in years with no KS2 baseline (e.g. 2024/25) — that is DfE policy,
-# not missing data.
+# ── KS4 National and LA Headlines (one data set, two streams) ────────────────
+# DfE's "summary, all state-funded" data set: England, regional and LA rows,
+# 2018/19 → latest. URL, measures and filters: ks4_summary.py.
 
-_KS4_NATIONAL_CSV_URL = (
-    "https://explore-education-statistics.service.gov.uk/data-catalogue/"
-    "data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv"
-)
+def _read_ks4_summary(logger):
+    """Download DfE's KS4 summary data set."""
+    import pandas as pd
 
-_KS4_NATIONAL_COL_MAP = {
-    "attainment8_average":    "attainment_8_score",
-    "progress8_average":      "progress_8_score",
-    "engmath_94_percent":     "english_maths_standard_pass_pct",
-    "engmath_95_percent":     "english_maths_strong_pass_pct",
-    "ebacc_entering_percent": "ebacc_entry_pct",
-    "ebacc_94_percent":       "ebacc_standard_pass_pct",
-    "ebacc_95_percent":       "ebacc_strong_pass_pct",
-    "ebacc_aps_average":      "ebacc_avg_score",
-}
+    logger.info("Downloading KS4 summary data set: %s", KS4_SUMMARY_CSV_URL)
+    resp = requests.get(KS4_SUMMARY_CSV_URL, timeout=60)
+    resp.raise_for_status()
+    return pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
 
 
 class EESKs4NationalStream(Stream):
-    """National KS4 headline averages — one row per academic year.
-
-    Filters to geographic_level == 'National', establishment_type_group ==
-    'All state-funded', breakdown_topic == 'Total', breakdown == 'Total'
-    so only the England-wide all-pupils row per year is emitted.
-    """
+    """National KS4 headline averages — one row per academic year (England,
+    all state-funded schools, all pupils)."""
 
     name = "ees_ks4_national"
     primary_keys = ["time_period"]
@@ -602,34 +598,41 @@ class EESKs4NationalStream(Stream):
 
     schema = th.PropertiesList(
         th.Property("time_period", th.StringType, required=True),
-        *[th.Property(out, th.StringType) for out in _KS4_NATIONAL_COL_MAP.values()],
+        *[th.Property(out, th.StringType) for out in KS4_HEADLINE_COL_MAP.values()],
     ).to_dict()
 
     def get_records(self, context):
-        import pandas as pd
-
-        self.logger.info("Downloading KS4 national headlines: %s", _KS4_NATIONAL_CSV_URL)
-        resp = requests.get(_KS4_NATIONAL_CSV_URL, timeout=60)
-        resp.raise_for_status()
-
-        df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
-        df.columns = [c.strip().lower() for c in df.columns]
-
-        for col, want in [
-            ("geographic_level", "national"),
-            ("establishment_type_group", "all state-funded"),
-            ("breakdown_topic", "total"),
-            ("breakdown", "total"),
-        ]:
-            if col in df.columns:
-                df = df[df[col].str.strip().str.lower() == want]
-
+        df = headline_rows(_read_ks4_summary(self.logger), "National")
         self.logger.info("Emitting %d national KS4 rows", len(df))
         for _, row in df.iterrows():
-            record = {"time_period": row.get("time_period", "").strip()}
-            for csv_col, field in _KS4_NATIONAL_COL_MAP.items():
-                record[field] = row.get(csv_col, "").strip()
-            yield record
+            yield headline_record(row, ("time_period",))
+
+
+class EESKs4LaStream(Stream):
+    """DfE's KS4 local-authority averages — one row per academic year and LA
+    (all state-funded schools, all pupils), from the same data set as
+    ees_ks4_national. They match DfE's published performance-table LA averages
+    and replace a mean the API took over every school, independent and special
+    included (audit H2). old_la_code is the GIAS LA code: schools join on it,
+    not on the name."""
+
+    name = "ees_ks4_la"
+    primary_keys = ["time_period", "old_la_code"]
+    replication_key = None
+
+    schema = th.PropertiesList(
+        th.Property("time_period", th.StringType, required=True),
+        th.Property("old_la_code", th.StringType, required=True),
+        th.Property("new_la_code", th.StringType),
+        th.Property("la_name", th.StringType),
+        *[th.Property(out, th.StringType) for out in KS4_HEADLINE_COL_MAP.values()],
+    ).to_dict()
+
+    def get_records(self, context):
+        df = headline_rows(_read_ks4_summary(self.logger), "Local authority")
+        self.logger.info("Emitting %d LA KS4 rows", len(df))
+        for _, row in df.iterrows():
+            yield headline_record(row, ("time_period", "old_la_code", "new_la_code", "la_name"))
 
 
 # ── Legacy KS2 (pre-COVID wide format from DfE performance tables) ────────────
@@ -972,6 +975,7 @@ class TapUKEES(Tap):
             LegacyKS4Stream(self),
             EESKs2NationalStream(self),
             EESKs4NationalStream(self),
+            EESKs4LaStream(self),
         ]
 
 

pipeline/tests/test_ees_ks4_info_names.py

@@ -0,0 +1,67 @@
+"""KS4 school information for 2023/24 exists only in the 2023/24 release,
+whose file uses DfE's older column names. The stream declared only the newer
+ones, so every 2023/24 field loaded as null (audit C2). The headers below are
+DfE's, copied from 202324_information_about_schools_final.csv and
+202425_information_about_schools_final.csv.
+"""
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-ees'
+          / 'tap_uk_ees' / 'ks4_info.py')
+
+HEADER_2023_24 = (
+    'time_period', 'time_identifier', 'geographic_level', 'country_code', 'country_name',
+    'school_laestab', 'school_urn', 'school_name', 'old_la_code', 'new_la_code', 'la_name',
+    'version', 'establishment_type_group', 'full_address', 'telnum', 'pcon_code', 'pcon_name',
+    'contflag', 'iclose', 'reldenom', 'admpol_pt', 'egender', 'feeder', 'agerange',
+    't_allks_pupils', 't_allks_boys', 't_allks_girls', 't_pupils', 't_boys', 'pt_boys',
+    't_girls', 'pt_girls', 'avg_ks2_scaledscore', 't_prior_lo', 'pt_prior_lo', 't_prior_av',
+    'pt_prior_av', 't_prior_hi', 'pt_prior_hi', 't_disadvantaged', 'pt_disadvantaged',
+    't_not_disadvantaged', 'pt_not_disadvantaged', 't_language_not_english',
+    'pt_language_not_english', 't_language_english', 'pt_language_english',
+    't_language_unknown', 'pt_language_unknown', 't_not_mobile', 'pt_not_mobile',
+    't_sen_with_ehcp', 'pt_sen_with_ehcp', 't_sen', 'pt_sen', 't_sen_no_ehcp',
+    'pt_sen_no_ehcp', 'diffn_att8', 'diffn_p8mea', 'p8_banding',
+)
+
+HEADER_2024_25 = (
+    'time_period', 'time_identifier', 'geographic_level', 'country_code', 'country_name',
+    'school_laestab', 'school_urn', 'school_name', 'old_la_code', 'new_la_code', 'la_name',
+    'version', 'establishment_type_group', 'full_address', 'telnum', 'pcon_code', 'pcon_name',
+    'contflag', 'iclose', 'reldenom', 'admpol_pt', 'egender', 'feeder', 'agerange',
+    'allks_pupil_count', 'allks_boys_count', 'allks_girls_count', 'endks4_pupil_count',
+    'ks2_scaledscore_average', 'sen_with_ehcp_pupil_count', 'sen_with_ehcp_pupil_percent',
+    'sen_pupil_count', 'sen_pupil_percent', 'sen_no_ehcp_pupil_count',
+    'sen_no_ehcp_pupil_percent', 'attainment8_diffn', 'progress8_diffn', 'progress8_banding',
+)
+
+
+@pytest.fixture
+def ks4_info():
+    spec = importlib.util.spec_from_file_location('ks4_info', MODULE)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_every_declared_field_is_in_the_2023_24_file_once_renamed(ks4_info):
+    renamed = {ks4_info.KS4_INFO_RENAMES.get(c, c) for c in HEADER_2023_24}
+    assert set(ks4_info.KS4_INFO_FIELDS) - renamed == set()
+
+
+def test_every_declared_field_is_in_the_2024_25_file(ks4_info):
+    assert set(ks4_info.KS4_INFO_FIELDS) - set(HEADER_2024_25) == set()
+
+
+def test_the_renames_cannot_collide_with_either_file(ks4_info):
+    # An old name in the current file, or a new name already in the old file,
+    # would let a rename overwrite a real column.
+    assert set(ks4_info.KS4_INFO_RENAMES) & set(HEADER_2024_25) == set()
+    assert set(ks4_info.KS4_INFO_RENAMES.values()) & set(HEADER_2023_24) == set()
+
+
+def test_every_rename_names_a_declared_field(ks4_info):
+    assert set(ks4_info.KS4_INFO_RENAMES.values()) <= set(ks4_info.KS4_INFO_FIELDS)

pipeline/tests/test_ees_ks4_summary.py

@@ -0,0 +1,72 @@
+"""DfE's KS4 "summary, all state-funded" data set holds England, regional and
+LA rows. The England stream kept only the England row; its LA rows match
+DfE's published LA averages exactly, where the API's own mean was 7 points
+low (audit H2). Values below are DfE's for Kensington and Chelsea (207) and
+Wandsworth (212).
+"""
+import importlib.util
+import io
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-ees'
+          / 'tap_uk_ees' / 'ks4_summary.py')
+
+CSV = """time_period,geographic_level,old_la_code,new_la_code,la_name,establishment_type_group,breakdown_topic,breakdown,attainment8_average,progress8_average,engmath_94_percent,engmath_95_percent,ebacc_entering_percent,ebacc_94_percent,ebacc_95_percent,ebacc_aps_average
+202425,National,,,,All state-funded,Total,Total,46.1,z,64.5,45.7,40.5,26.9,17.7,4.1
+202425,National,,,,All state-funded,Sex,Boys,44.1,z,62.0,43.0,38.0,24.0,16.0,3.9
+202425,Regional,,,,All state-funded,Total,Total,47.2,z,66.0,47.0,41.0,28.0,18.0,4.2
+202425,Local authority,207,E09000020,Kensington and Chelsea,All state-funded,Total,Total,54.5,z,77,61.4,45.6,32,26.6,4.89
+202425,Local authority,207,E09000020,Kensington and Chelsea,All state-funded,Sex,Girls,57.0,z,80,64.0,48.0,35,28.0,5.1
+202324,Local authority,207,E09000020,Kensington and Chelsea,All state-funded,Total,Total,54.5,0.29,76,60.0,44.0,31,25.0,4.8
+202425,Local authority,212,E09000032,Wandsworth,All state-funded,Total,Total,51.8,z,72,55.0,50.0,33,24.0,4.6
+202425,Local authority,212,E09000032,Wandsworth,Academies and free schools,Total,Total,52.0,z,73,56.0,51.0,34,25.0,4.7
+"""
+
+LA_KEYS = ('time_period', 'old_la_code', 'new_la_code', 'la_name')
+
+
+def _df():
+    return pd.read_csv(io.StringIO(CSV), dtype=str, keep_default_na=False)
+
+
+@pytest.fixture
+def summary():
+    spec = importlib.util.spec_from_file_location('ks4_summary', MODULE)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_la_rows_are_one_per_la_and_year(summary):
+    rows = summary.headline_rows(_df(), 'Local authority')
+    assert sorted(zip(rows['time_period'], rows['old_la_code'])) == [
+        ('202324', '207'), ('202425', '207'), ('202425', '212')]
+
+
+def test_an_la_record_carries_codes_name_and_the_headline_measures(summary):
+    rows = summary.headline_rows(_df(), 'Local authority')
+    row = rows[(rows['time_period'] == '202425') & (rows['old_la_code'] == '207')].iloc[0]
+    assert summary.headline_record(row, LA_KEYS) == {
+        'time_period': '202425', 'old_la_code': '207', 'new_la_code': 'E09000020',
+        'la_name': 'Kensington and Chelsea',
+        'attainment_8_score': '54.5', 'progress_8_score': 'z',
+        'english_maths_standard_pass_pct': '77', 'english_maths_strong_pass_pct': '61.4',
+        'ebacc_entry_pct': '45.6', 'ebacc_standard_pass_pct': '32',
+        'ebacc_strong_pass_pct': '26.6', 'ebacc_avg_score': '4.89',
+    }
+
+
+def test_the_england_rows_are_one_per_year(summary):
+    rows = summary.headline_rows(_df(), 'National')
+    assert list(rows['time_period']) == ['202425']
+    assert summary.headline_record(rows.iloc[0], ('time_period',))['attainment_8_score'] == '46.1'
+
+
+def test_column_names_and_labels_match_whatever_their_case(summary):
+    df = _df()
+    df.columns = [c.upper() for c in df.columns]
+    df['GEOGRAPHIC_LEVEL'] = df['GEOGRAPHIC_LEVEL'].str.upper()
+    assert len(summary.headline_rows(df, 'Local authority')) == 3

pipeline/tests/test_ees_release_precedence.py

@@ -0,0 +1,100 @@
+"""DfE re-publishes earlier years inside later KS4 releases.
+
+The 2024/25 results file holds 2022/23, 2023/24 and 2024/25 under current
+column names. The 2023/24 release's own file, re-issued in March 2026 under
+older names, was read after it and overwrote every 2023/24 row with blanks
+(audit C2). For the KS4 results stream the newest release owns every year it
+contains.
+"""
+import importlib.util
+import re
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+TAP_DIR = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-ees'
+           / 'tap_uk_ees')
+
+
+@pytest.fixture
+def precedence():
+    spec = importlib.util.spec_from_file_location(
+        'release_precedence', TAP_DIR / 'release_precedence.py')
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _release(period):
+    return {'id': f'release-{period}', 'time_period': period}
+
+
+def test_releases_are_taken_newest_first_whatever_order_the_api_gives(precedence):
+    releases = [_release('202223'), _release('202425'), _release(None), _release('202324')]
+    ordered = precedence.newest_first(releases)
+    assert [r['time_period'] for r in ordered] == ['202425', '202324', '202223', None]
+
+
+def test_a_year_a_newer_release_supplied_is_dropped_from_an_older_one(precedence):
+    newer = pd.DataFrame({'time_period': ['202425', '202324', '202223'], 'school_urn': ['1'] * 3})
+    older = pd.DataFrame({'time_period': ['202324', '202324', '201920'], 'school_urn': ['1', '2', '1']})
+
+    kept, skipped = precedence.drop_owned_periods(older, precedence.periods_in(newer))
+
+    assert list(kept['time_period']) == ['201920']
+    assert skipped == {'202324': 2}
+
+
+def test_periods_match_despite_surrounding_spaces(precedence):
+    owned = precedence.periods_in(pd.DataFrame({'time_period': [' 202324 ']}))
+    kept, skipped = precedence.drop_owned_periods(pd.DataFrame({'time_period': ['202324']}), owned)
+    assert owned == {'202324'}
+    assert kept.empty
+    assert skipped == {'202324': 1}
+
+
+def test_nothing_is_dropped_before_any_year_is_owned(precedence):
+    df = pd.DataFrame({'time_period': ['202324'], 'school_urn': ['1']})
+    kept, skipped = precedence.drop_owned_periods(df, set())
+    assert kept.equals(df)
+    assert skipped == {}
+
+
+def test_a_file_without_time_period_is_left_alone(precedence):
+    df = pd.DataFrame({'school_urn': ['1']})
+    kept, skipped = precedence.drop_owned_periods(df, {'202324'})
+    assert kept.equals(df)
+    assert skipped == {}
+    assert precedence.periods_in(df) == set()
+
+
+def test_only_the_ks4_results_stream_opts_in():
+    # A general rule would wipe KS2: the 2024/25 KS2 file holds 98,448 of the
+    # 955,956 rows the 2023/24 release has for 2023/24.
+    source = (TAP_DIR / 'tap.py').read_text()
+    opted_in = [chunk.split('(')[0] for chunk in source.split('\nclass ')[1:]
+                if re.search(r'_newest_release_owns_period\s*=\s*True', chunk)]
+    assert opted_in == ['EESKS4PerformanceStream']
+
+
+@pytest.mark.parametrize('slug, period', [
+    ('2024-25', '202425'),
+    ('2024-25-revised', '202425'),
+    ('2025-26-provisional', '202526'),
+    ('latest', None),
+    ('', None),
+])
+def test_a_release_slug_gives_its_year_whatever_its_suffix(precedence, slug, period):
+    # KS2 already publishes "2024-25-revised" and "2025-26-provisional". An
+    # unread suffix sent the release last, behind the provisional one for the
+    # same year, which then owned the year and dropped every revised row.
+    assert precedence.slug_to_time_period(slug) == period
+
+
+def test_within_a_year_the_api_order_is_kept(precedence):
+    # The API lists a year's revised release before its first release.
+    releases = [{'id': 'revised', 'time_period': '202526'},
+                {'id': 'first', 'time_period': '202526'},
+                {'id': 'older', 'time_period': '202425'}]
+    assert [r['id'] for r in precedence.newest_first(releases)] == ['revised', 'first', 'older']

pipeline/transform/macros/safe_numeric.sql

@@ -3,7 +3,9 @@
   Casts a string column to numeric, treating any non-numeric value as NULL.
   Handles all EES suppression codes (z, c, x, q, u, etc.) without needing
   an explicit list — any string that doesn't look like a number becomes NULL.
+  A trailing percent sign is accepted: DfE's 2023/24 KS2 information file
+  writes percentages as "34%" (audit C2).
 #}
 {% macro safe_numeric(col) -%}
-    CASE WHEN {{ col }} ~ '^-?[0-9]+(\.[0-9]+)?$' THEN {{ col }}::numeric ELSE NULL END
+    CASE WHEN {{ col }} ~ '^-?[0-9]+(\.[0-9]+)?%?$' THEN rtrim({{ col }}, '%')::numeric ELSE NULL END
 {%- endmacro %}

pipeline/transform/models/marts/_marts_schema.yml

@@ -279,11 +279,24 @@ models:
         tests: [not_null, unique]
 
   - name: fact_ks4_national_averages
-    description: Computed national KS4 averages (means across state schools in our dataset — not official DfE figures) — one row per academic year
+    description: Official DfE KS4 national headline averages (England, all state-funded schools, all pupils) — one row per academic year
     columns:
       - name: year
         tests: [not_null, unique]
 
+  - name: fact_ks4_la_averages
+    description: Official DfE KS4 local-authority averages (all state-funded schools, all pupils) — one row per academic year and LA; la_code is the GIAS LA code
+    columns:
+      - name: year
+        tests: [not_null]
+      - name: la_code
+        tests: [not_null]
+      - name: la_name
+        tests: [not_null]
+    tests:
+      - unique:
+          column_name: "year || '-' || la_code"
+
   - name: fact_deprivation
     description: IDACI deprivation index — one row per URN
     columns:

pipeline/transform/models/marts/fact_ks4_la_averages.sql

@@ -0,0 +1,22 @@
+{{ config(materialized='table') }}
+
+-- Mart: OFFICIAL DfE KS4 local-authority averages — one row per academic year
+-- and LA (all state-funded schools, all pupils), from the same EES data set
+-- as fact_ks4_national_averages. Feeds "vs LA avg" on search rows and map
+-- cards, replacing a mean the API took over every school in the LA,
+-- independent and special included (audit H2). la_code is the GIAS LA code.
+
+select
+    year,
+    la_code,
+    la_name,
+    attainment_8_score,
+    progress_8_score,
+    english_maths_standard_pass_pct,
+    english_maths_strong_pass_pct,
+    ebacc_entry_pct,
+    ebacc_standard_pass_pct,
+    ebacc_strong_pass_pct,
+    ebacc_avg_score
+from {{ ref('stg_ees_ks4_la') }}
+order by year, la_code

pipeline/transform/models/staging/_stg_sources.yml

@@ -79,6 +79,9 @@ sources:
       - name: ees_ks4_national
         description: Official KS4 national headline averages from DfE EES data catalogue — one row per academic year
 
+      - name: ees_ks4_la
+        description: Official KS4 local-authority averages (all state-funded schools) from the same DfE EES data set as ees_ks4_national — one row per academic year and LA
+
       # Phonics: no school-level data on EES (only national/LA level)
 
       - name: fbit_finance

pipeline/transform/models/staging/stg_ees_ks2.yml

@@ -0,0 +1,22 @@
+version: 2
+
+unit_tests:
+  - name: stg_ees_ks2_reads_percentages_written_with_a_sign
+    description: >
+      DfE's 2023/24 KS2 information file writes percentages as "34%", and every
+      2023/24 percentage loaded as null (audit C2). Plain numbers still load;
+      suppression codes stay null. School 147411's 2023/24 figures are DfE's.
+    model: stg_ees_ks2
+    given:
+      - input: source('raw', 'ees_ks2_attainment')
+        rows:
+          - {school_urn: '147411', time_period: '202324', subject: 'Reading, writing and maths', breakdown_topic: 'All pupils', breakdown: 'Total', expected_standard_pupil_percent: '70'}
+          - {school_urn: '100000', time_period: '202425', subject: 'Reading, writing and maths', breakdown_topic: 'All pupils', breakdown: 'Total', expected_standard_pupil_percent: '79'}
+      - input: source('raw', 'ees_ks2_info')
+        rows:
+          - {school_urn: '147411', time_period: '202324', totpups: '818', telig: '112', ptfsm6cla1a: '34%', ptealgrp2: '56%', psenelk: '20%', psenele: 'c', ptmobn: '87.5%'}
+          - {school_urn: '100000', time_period: '202425', totpups: '792', telig: '105', ptfsm6cla1a: '32', ptealgrp2: 'x', psenelk: '14', psenele: '2', ptmobn: '87'}
+    expect:
+      rows:
+        - {urn: 147411, year: 202324, total_pupils: 818, rwm_expected_pct: 70, disadvantaged_pct: 34, eal_pct: 56, sen_support_pct: 20, sen_ehcp_pct: null, stability_pct: 87.5}
+        - {urn: 100000, year: 202425, total_pupils: 792, rwm_expected_pct: 79, disadvantaged_pct: 32, eal_pct: null, sen_support_pct: 14, sen_ehcp_pct: 2, stability_pct: 87}

pipeline/transform/models/staging/stg_ees_ks4_la.sql

@@ -0,0 +1,21 @@
+-- Staging model: official DfE KS4 local-authority averages — one row per
+-- academic year and LA (all state-funded schools, all pupils). Same EES data
+-- set as stg_ees_ks4_national. la_code is the GIAS LA code
+-- (dim_location.local_authority_code). Suppressed values ('z', 'x') are
+-- coerced to NULL by safe_numeric.
+
+select
+    cast(trim(time_period) as integer)                        as year,
+    cast(trim(old_la_code) as integer)                        as la_code,
+    trim(la_name)                                             as la_name,
+    {{ safe_numeric('attainment_8_score') }}                  as attainment_8_score,
+    {{ safe_numeric('progress_8_score') }}                    as progress_8_score,
+    {{ safe_numeric('english_maths_standard_pass_pct') }}     as english_maths_standard_pass_pct,
+    {{ safe_numeric('english_maths_strong_pass_pct') }}       as english_maths_strong_pass_pct,
+    {{ safe_numeric('ebacc_entry_pct') }}                     as ebacc_entry_pct,
+    {{ safe_numeric('ebacc_standard_pass_pct') }}             as ebacc_standard_pass_pct,
+    {{ safe_numeric('ebacc_strong_pass_pct') }}               as ebacc_strong_pass_pct,
+    {{ safe_numeric('ebacc_avg_score') }}                     as ebacc_avg_score
+from {{ source('raw', 'ees_ks4_la') }}
+where time_period ~ '^[0-9]+$'
+  and old_la_code ~ '^[0-9]+$'

pipeline/transform/models/staging/stg_ees_ks4_la.yml

@@ -0,0 +1,15 @@
+version: 2
+
+unit_tests:
+  - name: stg_ees_ks4_la_casts_codes_years_and_measures
+    description: DfE's LA rows, keyed by the GIAS LA code. Suppressed values ('z') become null.
+    model: stg_ees_ks4_la
+    given:
+      - input: source('raw', 'ees_ks4_la')
+        rows:
+          - {time_period: '202425', old_la_code: '207', new_la_code: 'E09000020', la_name: 'Kensington and Chelsea', attainment_8_score: '54.5', progress_8_score: 'z', english_maths_standard_pass_pct: '77'}
+          - {time_period: '202324', old_la_code: '207', new_la_code: 'E09000020', la_name: 'Kensington and Chelsea', attainment_8_score: '54.5', progress_8_score: '0.29', english_maths_standard_pass_pct: '76'}
+    expect:
+      rows:
+        - {year: 202425, la_code: 207, la_name: 'Kensington and Chelsea', attainment_8_score: 54.5, progress_8_score: null, english_maths_standard_pass_pct: 77}
+        - {year: 202324, la_code: 207, la_name: 'Kensington and Chelsea', attainment_8_score: 54.5, progress_8_score: 0.29, english_maths_standard_pass_pct: 76}

pipeline/transform/tests/assert_ks2_info_percentages_loaded.sql

@@ -0,0 +1,16 @@
+-- Where a year's KS2 school information loaded (pupil counts present), its
+-- percentages loaded too. DfE's files give a disadvantaged % for 96–97% of
+-- those schools; 2023/24 loaded none because the file writes "34%" (audit
+-- C2). A year without an information file (2022/23) has no pupil counts and
+-- is skipped.
+
+select
+    year,
+    count(total_pupils) as with_pupils,
+    count(*) filter (where total_pupils is not null and disadvantaged_pct is not null)
+        as with_disadvantaged
+from {{ ref('stg_ees_ks2') }}
+group by year
+having count(total_pupils) >= 1000
+   and count(*) filter (where total_pupils is not null and disadvantaged_pct is not null)
+       < 0.9 * count(total_pupils)

pipeline/transform/tests/assert_ks4_la_averages_cover_las.sql

@@ -0,0 +1,13 @@
+-- DfE publishes an average for about 152 LAs a year. Fewer than 145 in the
+-- latest year (or none at all) means the LA filter in tap_uk_ees
+-- (ks4_summary.headline_rows) stopped matching, e.g. after DfE renamed a label.
+
+with latest as (
+    select max(year) as year from {{ ref('fact_ks4_la_averages') }}
+)
+
+select l.year, count(f.la_code) as las
+from latest l
+left join {{ ref('fact_ks4_la_averages') }} f on f.year = l.year
+group by l.year
+having count(f.la_code) < 145

pipeline/transform/tests/assert_ks4_la_averages_current.sql

@@ -0,0 +1,20 @@
+{{ config(severity='warn') }}
+
+-- DfE's LA averages cover the latest year with school results. They come from
+-- a fixed EES data set (ks4_summary.KS4_SUMMARY_CSV_URL); when DfE publishes a
+-- new year under a new data set, the school results move on and this mart does
+-- not, so /api/la-averages serves an empty map and every "vs LA avg" goes.
+-- A warning, not an error: it must not hold back a new year's school results.
+
+with schools as (
+    select max(year) as year from {{ ref('stg_ees_ks4') }} where attainment_8_score is not null
+),
+
+las as (
+    select max(year) as year from {{ ref('fact_ks4_la_averages') }}
+)
+
+select s.year as latest_school_year, l.year as latest_la_year
+from schools s
+cross join las l
+where l.year is null or l.year < s.year

pipeline/transform/tests/assert_ks4_years_have_results.sql

@@ -0,0 +1,12 @@
+-- Every year loaded from EES has an Attainment 8 for at least half its
+-- schools. DfE's files reach 82% each year; 2023/24 loaded at 0% after DfE
+-- re-issued its file under older column names (audit C2). Pre-2019 years come
+-- from stg_legacy_ks4 and are not checked here.
+
+select
+    year,
+    count(*) as schools,
+    count(attainment_8_score) as with_attainment_8
+from {{ ref('stg_ees_ks4') }}
+group by year
+having count(attainment_8_score) < 0.5 * count(*)