From 34a5de2687921d0f8dab07f04862aae165cb9590 Mon Sep 17 00:00:00 2001
From: Tudor <tudor@Tudors-MacBook-Pro.local>
Date: Sun, 12 Jul 2026 21:55:34 +0100
Subject: [PATCH] feat(pipeline): Progress 8 banding and KS4 disadvantage gaps
 in marts

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB
---
 .../models/intermediate/int_ks4_with_lineage.sql       |  6 ++++--
 pipeline/transform/models/marts/_marts_schema.yml      |  8 ++++++++
 .../transform/models/marts/fact_ks4_performance.sql    |  3 +++
 pipeline/transform/models/staging/stg_ees_ks4.sql      | 10 ++++++++--
 pipeline/transform/models/staging/stg_legacy_ks4.sql   |  7 ++++++-
 5 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql b/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql
index 4156d0e..9625ba7 100644
--- a/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql
+++ b/pipeline/transform/models/intermediate/int_ks4_with_lineage.sql
@@ -18,7 +18,8 @@ current_ks4 as (
         english_maths_strong_pass_pct, english_maths_standard_pass_pct,
         ebacc_entry_pct, ebacc_strong_pass_pct, ebacc_standard_pass_pct, ebacc_avg_score,
         gcse_grade_91_pct,
-        sen_pct, sen_support_pct, sen_ehcp_pct
+        sen_pct, sen_support_pct, sen_ehcp_pct,
+        progress_8_banding, attainment_8_disadvantage_gap, progress_8_disadvantage_gap
     from all_ks4
 ),
 
@@ -34,7 +35,8 @@ predecessor_ks4 as (
         ks4.english_maths_strong_pass_pct, ks4.english_maths_standard_pass_pct,
         ks4.ebacc_entry_pct, ks4.ebacc_strong_pass_pct, ks4.ebacc_standard_pass_pct, ks4.ebacc_avg_score,
         ks4.gcse_grade_91_pct,
-        ks4.sen_pct, ks4.sen_support_pct, ks4.sen_ehcp_pct
+        ks4.sen_pct, ks4.sen_support_pct, ks4.sen_ehcp_pct,
+        ks4.progress_8_banding, ks4.attainment_8_disadvantage_gap, ks4.progress_8_disadvantage_gap
     from all_ks4 ks4
     inner join {{ ref('int_school_lineage') }} lin
         on ks4.urn = lin.predecessor_urn
diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml
index 3ab4044..58bb7b8 100644
--- a/pipeline/transform/models/marts/_marts_schema.yml
+++ b/pipeline/transform/models/marts/_marts_schema.yml
@@ -104,6 +104,14 @@ models:
         tests: [not_null]
       - name: year
         tests: [not_null]
+      - name: progress_8_banding
+        tests:
+          - accepted_values:
+              values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average']
+              config:
+                where: "progress_8_banding is not null"
+      - name: attainment_8_disadvantage_gap
+      - name: progress_8_disadvantage_gap
     tests:
       - unique:
           column_name: "urn || '-' || year"
diff --git a/pipeline/transform/models/marts/fact_ks4_performance.sql b/pipeline/transform/models/marts/fact_ks4_performance.sql
index 4607dfb..57de61b 100644
--- a/pipeline/transform/models/marts/fact_ks4_performance.sql
+++ b/pipeline/transform/models/marts/fact_ks4_performance.sql
@@ -16,6 +16,9 @@ select
     progress_8_score,
     progress_8_lower_ci,
     progress_8_upper_ci,
+    progress_8_banding,
+    attainment_8_disadvantage_gap,
+    progress_8_disadvantage_gap,
     progress_8_english,
     progress_8_maths,
     progress_8_ebacc,
diff --git a/pipeline/transform/models/staging/stg_ees_ks4.sql b/pipeline/transform/models/staging/stg_ees_ks4.sql
index 7cd1d3e..7395ebc 100644
--- a/pipeline/transform/models/staging/stg_ees_ks4.sql
+++ b/pipeline/transform/models/staging/stg_ees_ks4.sql
@@ -62,7 +62,10 @@ info as (
         {{ safe_numeric('ks2_scaledscore_average') }} as prior_attainment_avg,
         {{ safe_numeric('sen_pupil_percent') }} as sen_pct,
         {{ safe_numeric('sen_with_ehcp_pupil_percent') }} as sen_ehcp_pct,
-        {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct
+        {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct,
+        nullif(trim(progress8_banding), '') as progress_8_banding,
+        {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap,
+        {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap
     from {{ source('raw', 'ees_ks4_info') }}
     where school_urn is not null
 )
@@ -102,7 +105,10 @@ select
     -- Context
     i.sen_pct,
     i.sen_ehcp_pct,
-    i.sen_support_pct
+    i.sen_support_pct,
+    i.progress_8_banding,
+    i.attainment_8_disadvantage_gap,
+    i.progress_8_disadvantage_gap
 
 from all_pupils p
 left join info i on p.urn = i.urn and p.year = i.year
diff --git a/pipeline/transform/models/staging/stg_legacy_ks4.sql b/pipeline/transform/models/staging/stg_legacy_ks4.sql
index 2254070..c6b384b 100644
--- a/pipeline/transform/models/staging/stg_legacy_ks4.sql
+++ b/pipeline/transform/models/staging/stg_legacy_ks4.sql
@@ -42,7 +42,12 @@ select
     -- SEN
     null::numeric                           as sen_pct,
     {{ safe_numeric('sen_support_pct') }}   as sen_support_pct,
-    {{ safe_numeric('sen_ehcp_pct') }}      as sen_ehcp_pct
+    {{ safe_numeric('sen_ehcp_pct') }}      as sen_ehcp_pct,
+
+    -- Progress 8 banding & disadvantage gaps (not published in legacy format)
+    null::text    as progress_8_banding,
+    null::numeric as attainment_8_disadvantage_gap,
+    null::numeric as progress_8_disadvantage_gap
 
 from {{ source('raw', 'legacy_ks4') }}
 where urn is not null