fix(data): official DfE KS4 national headline averages; drop mislabelled computed means

New ees_ks4_national stream ingests the EES 'National characteristics
summary data' series (England, state-funded, all pupils). The old mart's
unweighted school means were 7-15 points off every headline measure and
produced an impossible national Progress 8 (-0.27). The API's computed
fallback is gone too: the footnote calls these figures official, so an
unbuilt mart now yields an empty series, never a stand-in.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB
This commit is contained in:
Tudor
2026-07-16 19:07:49 +01:00
co-authored by Claude Fable 5
parent 1d855f3c17
commit c9e324635b
14 changed files with 475 additions and 51 deletions
@@ -564,6 +564,74 @@ class EESKs2NationalStream(Stream):
yield record
# ── KS4 National Headlines (national level only — one row per year) ──────────
# Dataset: "National characteristics summary data" (Key stage 4 performance).
# Official England state-funded headline measures, 2018/19 → latest.
# Suppressed values ('z', 'x') → NULL downstream. Progress 8 is legitimately
# absent in years with no KS2 baseline (e.g. 2024/25) — that is DfE policy,
# not missing data.
_KS4_NATIONAL_CSV_URL = (
"https://explore-education-statistics.service.gov.uk/data-catalogue/"
"data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv"
)
_KS4_NATIONAL_COL_MAP = {
"attainment8_average": "attainment_8_score",
"progress8_average": "progress_8_score",
"engmath_94_percent": "english_maths_standard_pass_pct",
"engmath_95_percent": "english_maths_strong_pass_pct",
"ebacc_entering_percent": "ebacc_entry_pct",
"ebacc_94_percent": "ebacc_standard_pass_pct",
"ebacc_95_percent": "ebacc_strong_pass_pct",
"ebacc_aps_average": "ebacc_avg_score",
}
class EESKs4NationalStream(Stream):
"""National KS4 headline averages — one row per academic year.
Filters to geographic_level == 'National', establishment_type_group ==
'All state-funded', breakdown_topic == 'Total', breakdown == 'Total'
so only the England-wide all-pupils row per year is emitted.
"""
name = "ees_ks4_national"
primary_keys = ["time_period"]
replication_key = None
schema = th.PropertiesList(
th.Property("time_period", th.StringType, required=True),
*[th.Property(out, th.StringType) for out in _KS4_NATIONAL_COL_MAP.values()],
).to_dict()
def get_records(self, context):
import pandas as pd
self.logger.info("Downloading KS4 national headlines: %s", _KS4_NATIONAL_CSV_URL)
resp = requests.get(_KS4_NATIONAL_CSV_URL, timeout=60)
resp.raise_for_status()
df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
df.columns = [c.strip().lower() for c in df.columns]
for col, want in [
("geographic_level", "national"),
("establishment_type_group", "all state-funded"),
("breakdown_topic", "total"),
("breakdown", "total"),
]:
if col in df.columns:
df = df[df[col].str.strip().str.lower() == want]
self.logger.info("Emitting %d national KS4 rows", len(df))
for _, row in df.iterrows():
record = {"time_period": row.get("time_period", "").strip()}
for csv_col, field in _KS4_NATIONAL_COL_MAP.items():
record[field] = row.get(csv_col, "").strip()
yield record
# ── Legacy KS2 (pre-COVID wide format from DfE performance tables) ────────────
# The DfE "Compare School Performance" site published school-level KS2 CSVs
# in a wide format (one row per school, ~300 columns). EES only has school-level
@@ -903,6 +971,7 @@ class TapUKEES(Tap):
LegacyKS2Stream(self),
LegacyKS4Stream(self),
EESKs2NationalStream(self),
EESKs4NationalStream(self),
]