fix(data): official DfE KS4 national headline averages; drop mislabelled computed means

New ees_ks4_national stream ingests the EES 'National characteristics
summary data' series (England, state-funded, all pupils). The old mart's
unweighted school means were 7-15 points off every headline measure and
produced an impossible national Progress 8 (-0.27). The API's computed
fallback is gone too: the footnote calls these figures official, so an
unbuilt mart now yields an empty series, never a stand-in.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB

pipeline/transform/models/marts/fact_ks4_national_averages.sql

@@ -1,25 +1,22 @@
 {{ config(materialized='table') }}
 
--- Mart: Computed national KS4 averages — one row per academic year.
--- Unlike fact_ks2_national_averages (official DfE figures), DfE publishes no
--- KS4 national-headline dataset we ingest yet, so these are means computed
--- across the state schools in our dataset. Computed once at build time so the
--- API never has to aggregate the full performance table per request.
--- Semantics match the API's previous per-request computation: rows where
--- attainment_8_score is non-null; per-column means ignore NULLs.
+-- Mart: OFFICIAL DfE KS4 national headline averages — one row per academic
+-- year (England, state-funded, all pupils), from the EES national dataset.
+-- Replaces the previous unweighted school-level means, which were 7–15
+-- points off every headline measure and produced an arithmetically
+-- impossible national Progress 8. gcse_grade_91_pct has no official
+-- national series and is NULL (schema kept for the API model).
 
 select
     year,
-    round(avg(attainment_8_score)::numeric, 2)               as attainment_8_score,
-    round(avg(progress_8_score)::numeric, 2)                 as progress_8_score,
-    round(avg(english_maths_standard_pass_pct)::numeric, 2)  as english_maths_standard_pass_pct,
-    round(avg(english_maths_strong_pass_pct)::numeric, 2)    as english_maths_strong_pass_pct,
-    round(avg(ebacc_entry_pct)::numeric, 2)                  as ebacc_entry_pct,
-    round(avg(ebacc_standard_pass_pct)::numeric, 2)          as ebacc_standard_pass_pct,
-    round(avg(ebacc_strong_pass_pct)::numeric, 2)            as ebacc_strong_pass_pct,
-    round(avg(ebacc_avg_score)::numeric, 2)                  as ebacc_avg_score,
-    round(avg(gcse_grade_91_pct)::numeric, 2)                as gcse_grade_91_pct
-from {{ ref('fact_ks4_performance') }}
-where attainment_8_score is not null
-group by year
+    attainment_8_score,
+    progress_8_score,
+    english_maths_standard_pass_pct,
+    english_maths_strong_pass_pct,
+    ebacc_entry_pct,
+    ebacc_standard_pass_pct,
+    ebacc_strong_pass_pct,
+    ebacc_avg_score,
+    cast(null as double precision) as gcse_grade_91_pct
+from {{ ref('stg_ees_ks4_national') }}
 order by year

pipeline/transform/models/staging/_stg_sources.yml

@@ -51,6 +51,9 @@ sources:
       - name: ees_ks2_national
         description: KS2 national headline averages from DfE EES data catalogue — one row per academic year
 
+      - name: ees_ks4_national
+        description: Official KS4 national headline averages from DfE EES data catalogue — one row per academic year
+
       # Phonics: no school-level data on EES (only national/LA level)
 
       - name: fbit_finance

pipeline/transform/models/staging/stg_ees_ks4_national.sql

@@ -0,0 +1,20 @@
+{{ config(materialized='table') }}
+
+-- Staging model: official DfE KS4 national headline averages — one row per
+-- academic year (England, all state-funded, all pupils). Source: EES data
+-- catalogue "National characteristics summary data". Suppressed values
+-- ('z', 'x') are coerced to NULL by safe_numeric — Progress 8 is 'z' in
+-- years with no KS2 baseline (e.g. 2024/25): legitimately unpublished.
+
+select
+    cast(trim(time_period) as integer)                        as year,
+    {{ safe_numeric('attainment_8_score') }}                  as attainment_8_score,
+    {{ safe_numeric('progress_8_score') }}                    as progress_8_score,
+    {{ safe_numeric('english_maths_standard_pass_pct') }}     as english_maths_standard_pass_pct,
+    {{ safe_numeric('english_maths_strong_pass_pct') }}       as english_maths_strong_pass_pct,
+    {{ safe_numeric('ebacc_entry_pct') }}                     as ebacc_entry_pct,
+    {{ safe_numeric('ebacc_standard_pass_pct') }}             as ebacc_standard_pass_pct,
+    {{ safe_numeric('ebacc_strong_pass_pct') }}               as ebacc_strong_pass_pct,
+    {{ safe_numeric('ebacc_avg_score') }}                     as ebacc_avg_score
+from {{ source('raw', 'ees_ks4_national') }}
+where time_period ~ '^[0-9]+$'