From 436ec6151bb09f20314d657d016b83bd3392e17c Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 08:42:40 +0100 Subject: [PATCH] fix(pipeline): thread KS2 progress CI columns through the legacy union and lineage model The AI review gate caught that stg_ees_ks2's 7 new columns broke the positional UNION ALL with stg_legacy_ks2 in int_ks2_with_lineage, and that the lineage CTEs never emitted them (same class of bug fixed for KS4 in 34a5de2). Legacy gets typed null placeholders at matching positions; both lineage CTEs pass the columns through. 45/45 columns verified name-identical in order across both union branches. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../models/intermediate/int_ks2_with_lineage.sql | 6 ++++++ pipeline/transform/models/staging/stg_legacy_ks2.sql | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql b/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql index 0fa7a72..c66bf5c 100644 --- a/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql +++ b/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql @@ -15,8 +15,11 @@ current_ks2 as ( year, total_pupils, eligible_pupils, rwm_expected_pct, rwm_high_pct, reading_expected_pct, reading_high_pct, reading_avg_score, reading_progress, + reading_progress_lower_ci, reading_progress_upper_ci, writing_expected_pct, writing_high_pct, writing_progress, + writing_progress_lower_ci, writing_progress_upper_ci, writing_working_towards_pct, maths_expected_pct, maths_high_pct, maths_avg_score, maths_progress, + maths_progress_lower_ci, maths_progress_upper_ci, gps_expected_pct, gps_high_pct, gps_avg_score, science_expected_pct, reading_absence_pct, writing_absence_pct, maths_absence_pct, gps_absence_pct, science_absence_pct, rwm_expected_boys_pct, rwm_high_boys_pct, rwm_expected_girls_pct, rwm_high_girls_pct, @@ -33,8 +36,11 @@ predecessor_ks2 as ( ks2.year, ks2.total_pupils, ks2.eligible_pupils, ks2.rwm_expected_pct, ks2.rwm_high_pct, ks2.reading_expected_pct, ks2.reading_high_pct, ks2.reading_avg_score, ks2.reading_progress, + ks2.reading_progress_lower_ci, ks2.reading_progress_upper_ci, ks2.writing_expected_pct, ks2.writing_high_pct, ks2.writing_progress, + ks2.writing_progress_lower_ci, ks2.writing_progress_upper_ci, ks2.writing_working_towards_pct, ks2.maths_expected_pct, ks2.maths_high_pct, ks2.maths_avg_score, ks2.maths_progress, + ks2.maths_progress_lower_ci, ks2.maths_progress_upper_ci, ks2.gps_expected_pct, ks2.gps_high_pct, ks2.gps_avg_score, ks2.science_expected_pct, ks2.reading_absence_pct, ks2.writing_absence_pct, ks2.maths_absence_pct, ks2.gps_absence_pct, ks2.science_absence_pct, ks2.rwm_expected_boys_pct, ks2.rwm_high_boys_pct, ks2.rwm_expected_girls_pct, ks2.rwm_high_girls_pct, diff --git a/pipeline/transform/models/staging/stg_legacy_ks2.sql b/pipeline/transform/models/staging/stg_legacy_ks2.sql index c719f20..637f39a 100644 --- a/pipeline/transform/models/staging/stg_legacy_ks2.sql +++ b/pipeline/transform/models/staging/stg_legacy_ks2.sql @@ -17,13 +17,23 @@ select {{ safe_numeric('reading_high_pct') }} as reading_high_pct, {{ safe_numeric('reading_avg_score') }} as reading_avg_score, {{ safe_numeric('reading_progress') }} as reading_progress, + -- Progress CIs / working-towards: not published in the legacy CSVs. + -- Typed placeholders keep positional alignment with stg_ees_ks2 in + -- int_ks2_with_lineage's UNION ALL. + null::numeric as reading_progress_lower_ci, + null::numeric as reading_progress_upper_ci, {{ safe_numeric('writing_expected_pct') }} as writing_expected_pct, {{ safe_numeric('writing_high_pct') }} as writing_high_pct, {{ safe_numeric('writing_progress') }} as writing_progress, + null::numeric as writing_progress_lower_ci, + null::numeric as writing_progress_upper_ci, + null::numeric as writing_working_towards_pct, {{ safe_numeric('maths_expected_pct') }} as maths_expected_pct, {{ safe_numeric('maths_high_pct') }} as maths_high_pct, {{ safe_numeric('maths_avg_score') }} as maths_avg_score, {{ safe_numeric('maths_progress') }} as maths_progress, + null::numeric as maths_progress_lower_ci, + null::numeric as maths_progress_upper_ci, {{ safe_numeric('gps_expected_pct') }} as gps_expected_pct, {{ safe_numeric('gps_high_pct') }} as gps_high_pct, {{ safe_numeric('gps_avg_score') }} as gps_avg_score,