fix(data): official DfE KS4 national headline averages; drop mislabelled computed means
New ees_ks4_national stream ingests the EES 'National characteristics summary data' series (England, state-funded, all pupils). The old mart's unweighted school means were 7-15 points off every headline measure and produced an impossible national Progress 8 (-0.27). The API's computed fallback is gone too: the footnote calls these figures official, so an unbuilt mart now yields an empty series, never a stand-in. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB
This commit is contained in:
@@ -564,6 +564,74 @@ class EESKs2NationalStream(Stream):
|
||||
yield record
|
||||
|
||||
|
||||
# ── KS4 National Headlines (national level only — one row per year) ──────────
|
||||
# Dataset: "National characteristics summary data" (Key stage 4 performance).
|
||||
# Official England state-funded headline measures, 2018/19 → latest.
|
||||
# Suppressed values ('z', 'x') → NULL downstream. Progress 8 is legitimately
|
||||
# absent in years with no KS2 baseline (e.g. 2024/25) — that is DfE policy,
|
||||
# not missing data.
|
||||
|
||||
_KS4_NATIONAL_CSV_URL = (
|
||||
"https://explore-education-statistics.service.gov.uk/data-catalogue/"
|
||||
"data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv"
|
||||
)
|
||||
|
||||
_KS4_NATIONAL_COL_MAP = {
|
||||
"attainment8_average": "attainment_8_score",
|
||||
"progress8_average": "progress_8_score",
|
||||
"engmath_94_percent": "english_maths_standard_pass_pct",
|
||||
"engmath_95_percent": "english_maths_strong_pass_pct",
|
||||
"ebacc_entering_percent": "ebacc_entry_pct",
|
||||
"ebacc_94_percent": "ebacc_standard_pass_pct",
|
||||
"ebacc_95_percent": "ebacc_strong_pass_pct",
|
||||
"ebacc_aps_average": "ebacc_avg_score",
|
||||
}
|
||||
|
||||
|
||||
class EESKs4NationalStream(Stream):
|
||||
"""National KS4 headline averages — one row per academic year.
|
||||
|
||||
Filters to geographic_level == 'National', establishment_type_group ==
|
||||
'All state-funded', breakdown_topic == 'Total', breakdown == 'Total'
|
||||
so only the England-wide all-pupils row per year is emitted.
|
||||
"""
|
||||
|
||||
name = "ees_ks4_national"
|
||||
primary_keys = ["time_period"]
|
||||
replication_key = None
|
||||
|
||||
schema = th.PropertiesList(
|
||||
th.Property("time_period", th.StringType, required=True),
|
||||
*[th.Property(out, th.StringType) for out in _KS4_NATIONAL_COL_MAP.values()],
|
||||
).to_dict()
|
||||
|
||||
def get_records(self, context):
|
||||
import pandas as pd
|
||||
|
||||
self.logger.info("Downloading KS4 national headlines: %s", _KS4_NATIONAL_CSV_URL)
|
||||
resp = requests.get(_KS4_NATIONAL_CSV_URL, timeout=60)
|
||||
resp.raise_for_status()
|
||||
|
||||
df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
|
||||
df.columns = [c.strip().lower() for c in df.columns]
|
||||
|
||||
for col, want in [
|
||||
("geographic_level", "national"),
|
||||
("establishment_type_group", "all state-funded"),
|
||||
("breakdown_topic", "total"),
|
||||
("breakdown", "total"),
|
||||
]:
|
||||
if col in df.columns:
|
||||
df = df[df[col].str.strip().str.lower() == want]
|
||||
|
||||
self.logger.info("Emitting %d national KS4 rows", len(df))
|
||||
for _, row in df.iterrows():
|
||||
record = {"time_period": row.get("time_period", "").strip()}
|
||||
for csv_col, field in _KS4_NATIONAL_COL_MAP.items():
|
||||
record[field] = row.get(csv_col, "").strip()
|
||||
yield record
|
||||
|
||||
|
||||
# ── Legacy KS2 (pre-COVID wide format from DfE performance tables) ────────────
|
||||
# The DfE "Compare School Performance" site published school-level KS2 CSVs
|
||||
# in a wide format (one row per school, ~300 columns). EES only has school-level
|
||||
@@ -903,6 +971,7 @@ class TapUKEES(Tap):
|
||||
LegacyKS2Stream(self),
|
||||
LegacyKS4Stream(self),
|
||||
EESKs2NationalStream(self),
|
||||
EESKs4NationalStream(self),
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user