From ccd8e73fe860f518211a923a04daf94dd624ead8 Mon Sep 17 00:00:00 2001 From: Tudor Date: Sun, 12 Jul 2026 22:02:28 +0100 Subject: [PATCH] fix(pipeline): include 2015/16 national averages Widen the year filter in stg_ees_ks2_national.sql from >= 201617 to >= 201516 so the England national-averages line no longer starts a year late; the catalogue CSV has a real, comparable 201516 row (2015/16 was the first year of the current expected-standard tests, so it's the correct floor). GPS/science/scaled-score national columns confirmed present at source with correct mapping; prod NULLs are stale raw data, backfilled by the next extract run. No _KS2_NATIONAL_COL_MAP change accompanies this fix. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../transform/models/staging/stg_ees_ks2_national.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipeline/transform/models/staging/stg_ees_ks2_national.sql b/pipeline/transform/models/staging/stg_ees_ks2_national.sql index 7335ce6..44dfbe4 100644 --- a/pipeline/transform/models/staging/stg_ees_ks2_national.sql +++ b/pipeline/transform/models/staging/stg_ees_ks2_national.sql @@ -31,4 +31,10 @@ select from {{ source('raw', 'ees_ks2_national') }} where time_period ~ '^[0-9]+$' - and cast(trim(time_period) as integer) >= 201617 + -- 2015/16 was the first year of the current expected-standard tests, so it's + -- the correct floor (not 2016/17 -- that excluded a real, comparable national + -- row). GPS/science/scaled-score columns are already mapped correctly end to + -- end (tap.py's _KS2_NATIONAL_COL_MAP + this model select them fine); the + -- prod NULLs for those fields are stale raw.ees_ks2_national data from before + -- the map covered them, not a mapping bug -- no map change accompanies this fix. + and cast(trim(time_period) as integer) >= 201516