ci: re-run PR checks (AI review job errored without posting findings)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB

docs/superpowers/specs/2026-07-11-compare-screen-redesign-design.md

@@ -252,12 +252,18 @@ mockups on 2026-07-12. The following are binding requirements for
 implementation, beyond what the mockups can show:
 
 1. **Chart truthfulness:** KS2 tests were cancelled 2019/20–2020/21
-   only; 2021/22 results exist and must be loaded (school-level rows
+   only. **2021/22 school-level figures are a permanent source gap** —
-   and the 2015/16 national figure, 53%, are currently missing from
+   DfE stated it would not publish KS2 2021/22 in performance tables
-   our dataset — pipeline task). Load official England averages for
+   (verified 2026-07-12 against EES, the CSP download service, and
-   GPS/science expected % and the scaled scores (reading/maths/GPS) —
+   DfE release notes; see `# TASK 6 VERIFICATION` in
-   the mockups' scaled-score ticks are indicative until then. Never
+   `pipeline/scripts/diagnose_compare_gaps.py`). The chart's England-
-   render missing years as if time were continuous.
+   only 2021/22 point with broken school lines is therefore the
+   correct permanent rendering; copy should say "DfE didn't publish
+   school-level figures for 2021/22", not "not in our dataset yet".
+   The 2015/16 national figure and the GPS/science/scaled-score
+   England averages ARE loadable (mapping already correct; refreshed
+   raw extract backfills them). Never render missing years as if time
+   were continuous.
 2. **Report-card summaries** count graded areas only — safeguarding is
    a separate binary flag, never included in rating counts.
 3. **Admissions:** use the real preference-breakdown fields from

pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py

@@ -68,6 +68,19 @@ COLUMN_PRIORITY = {
     "ungraded_inspection_date": [
         "Date of latest ungraded inspection",
     ],
+    # Report Card fields (post-Nov 2025 framework). Confirmed verbatim MI
+    # headers per diagnose_compare_gaps.py's Task 1(c) findings. No MI column
+    # currently exists for early-years or sixth-form report-card grades, so
+    # those two fields are deliberately omitted here (see schema below) --
+    # they stay absent from every record, same as the existing `report_url`
+    # pattern for fields with no COLUMN_PRIORITY entry.
+    "rc_safeguarding_met": ["Safeguarding standards"],
+    "rc_inclusion": ["Inclusion"],
+    "rc_curriculum_teaching": ["Curriculum and teaching"],
+    "rc_achievement": ["Achievement"],
+    "rc_attendance_behaviour": ["Attendance and behaviour"],
+    "rc_personal_development": ["Personal development and wellbeing"],
+    "rc_leadership_governance": ["Leadership and governance"],
 }
 
 
@@ -111,6 +124,17 @@ class OfstedInspectionsStream(Stream):
         th.Property("sixth_form_provision", th.StringType),
         th.Property("ungraded_outcome", th.StringType),
         th.Property("ungraded_inspection_date", th.StringType),
+        th.Property("rc_safeguarding_met", th.StringType),
+        th.Property("rc_inclusion", th.StringType),
+        th.Property("rc_curriculum_teaching", th.StringType),
+        th.Property("rc_achievement", th.StringType),
+        th.Property("rc_attendance_behaviour", th.StringType),
+        th.Property("rc_personal_development", th.StringType),
+        th.Property("rc_leadership_governance", th.StringType),
+        # No MI column exists for these yet; declared for forward
+        # compatibility with the mart schema, always emitted as absent/NULL.
+        th.Property("rc_early_years", th.StringType),
+        th.Property("rc_sixth_form", th.StringType),
         th.Property("report_url", th.StringType),
     ).to_dict()
 

pipeline/scripts/diagnose_compare_gaps.py

@@ -10,7 +10,6 @@ is not part of the repo's default environment, hence the `uv run --with`.)
 import io
 import re
 import sys
-import zipfile
 
 import pandas as pd
 import requests
@@ -39,7 +38,11 @@ def check_national_gps_science():
         print(f"  {csv_col}: {status}")
     gps_like = [c for c in df.columns if "gps" in c or "scita" in c or "sci" in c]
     print(f"  all gps/science-ish columns: {gps_like}")
-    nat = df[df.get("geographic_level", "").str.strip().str.lower() == "national"]
+    if "geographic_level" in df.columns:
+        nat = df[df["geographic_level"].str.strip().str.lower() == "national"]
+    else:
+        print("  geographic_level column missing — cannot isolate national rows")
+        return
     print(f"  national rows time_periods: {sorted(nat['time_period'].unique())}")
     # Sample the values our map would read for the latest year
     latest = nat[nat["time_period"] == nat["time_period"].max()]
@@ -169,7 +172,7 @@ if __name__ == "__main__":
 #       https://assets.publishing.service.gov.uk/media/6a27c45be13080622db38815/
 #       Management_information_-_state-funded_schools_-_latest_inspections_as_at_31_May_2026.csv
 #     This is a real .csv (not .ods) so section (c) ran to completion.
-#     Exact report-card column headers (6 grade columns + their paired date
+#     Exact report-card column headers (7 grade columns + their paired date
 #     columns, all present verbatim, case/spacing exactly as below):
 #       'Safeguarding standards'                 / 'Safeguarding standards - date of grade'
 #       'Inclusion'                               / 'Inclusion - date of grade'
@@ -184,9 +187,119 @@ if __name__ == "__main__":
 #     These are the new-style "report card" single-word-area grades
 #     (introduced alongside the "Attendance and behaviour" split from
 #     "Personal development"); they coexist in the same CSV with the legacy
-#     4-judgement OEIF columns ('Latest OEIF overall effectiveness',
+#     5-judgement OEIF columns ('Latest OEIF overall effectiveness',
 #     'Latest OEIF quality of education', 'Latest OEIF behaviour and
 #     attitudes', 'Latest OEIF personal development', 'Latest OEIF
 #     effectiveness of leadership and management'). Task 7 should map the 7
 #     report-card columns above (grade + date pairs, 6 of them, plus the
 #     safeguarding-effective flag) rather than inventing new column names.
+
+# TASK 6 VERIFICATION 2026-07-12: 2021/22 legacy KS2 school-level archive
+#
+# RESULT: BLOCKED at the source-data level. School-level KS2 attainment for
+# academic year 2021/22 was never published anywhere publicly by DfE -- not
+# in EES (confirmed by Task 1's finding (b) above), not in the legacy
+# "Compare School Performance" download wizard, and not as a standalone
+# performance-tables archive/ODS on assets.publishing.service.gov.uk. This
+# is a deliberate DfE decision, not a gap in our extraction logic.
+#
+# Confirming quote (Key stage 2 attainment 2021/22 release notes, via
+# https://explore-education-statistics.service.gov.uk/find-statistics/
+# key-stage-2-attainment/2021-22):
+#   "We will not publish key stage 2 data for academic year 2021/22 in
+#   performance tables (also known as Compare School and College
+#   Performance)." ... "The Department will, however, still produce the
+#   normal suite of key stage 2 accountability measures at school and
+#   multi-academy trust level and share these securely with primary
+#   schools, academy trusts and local authorities to inform school
+#   improvement discussions."
+# (i.e. school-level 202122 KS2 results exist internally at DfE but were
+# withheld from every public channel: performance tables/CSCP, EES, and by
+# extension the legacy DfE archives the current legacy_ks2_urls entries in
+# meltano.yml were sourced from.)
+#
+# What was tried:
+#  1. Direct download URL pattern from the task brief:
+#       https://www.compare-school-performance.service.gov.uk/download-data?download=true&regions=0&filters=KS2&fileformat=csv&year=2021-2022&meta=false
+#     -> HTTP 404, HTML error page (not a CSV/ZIP). Saved response inspected;
+#     confirmed 404 via response headers (`content-type: text/html`).
+#  2. Walked the actual multi-step download wizard at
+#     https://www.compare-school-performance.service.gov.uk/download-data
+#     with a browser User-Agent and a cookie jar, replicating the GET-based
+#     form steps: currentstep=year (downloadYear=2021-2022) -> currentstep=
+#     region (regiontype=all&la=0) -> currentstep=datatypes. On the final
+#     "datatypes" step, the checkbox list for 2021-2022 has NO "ks2" (or
+#     "ks2mats") option at all -- only ks4/ks4prov/ks4underlying/ks5* /
+#     pupil-destination/absence/census/mats checkboxes are present.
+#     Control check: repeating the same wizard walk for downloadYear=
+#     2018-2019, 2022-2023 and 2023-2024 shows a "ks2" (and "ks2mats")
+#     checkbox present in all three; downloadYear=2020-2021 (COVID-cancelled
+#     KS2 SATs year) also has NO ks2 checkbox, matching the pattern for a
+#     year where school-level KS2 genuinely isn't published. 2021-2022
+#     behaves identically to the cancelled 2020-2021 year, not like the
+#     normal 2018-2019/2022-2023/2023-2024 years.
+#  3. Web search for a standalone KS2 2022 performance-tables archive
+#     (e.g. "england_ks2final" for 2022) on assets.publishing.service.gov.uk
+#     found no such file; only unrelated 2022/2023-dated documents.
+#
+#  No ZIP was ever obtained -- /tmp/dfe-2021-2022-ks2.zip contains the 404
+#  HTML error page from attempt (1) above, not a real archive. It contains
+#  no england_ks2final.csv (there is no ZIP to look inside).
+#
+# Column-map check (brief's Step 1): NOT RUN -- there is no 2021/22
+# england_ks2final.csv to check headers against. This is moot until/unless
+# a non-public source (e.g. a manual/internal DfE extract) becomes
+# available; _LEGACY_KS2_COLUMN_MAP itself is unchanged and untested here.
+#
+# Recommendation: mark 202122 school-level KS2 as a genuine, permanent
+# source-data gap (not a backfill candidate) unless the project can obtain
+# the internal DfE extract DfE says it shared "securely with primary
+# schools, academy trusts and local authorities" -- that is not a route
+# available to this pipeline. Task 6's meltano.yml change (Step 2) and the
+# filebrowser upload should NOT proceed for 202122; there is nothing to
+# upload.
+
+# TASK 7 VALUE SAMPLE 2026-07-12: live value_counts() over the 7 report-card
+# columns (plus the related safeguarding-effective flag) in the same MI CSV
+# resolved by discover_csv_url() as at run time (31 May 2026 inspections
+# file). Blank cells read as the literal string 'NULL' (matches
+# keep_default_na=False in tap.py). Observed non-blank values, verbatim:
+#
+#   'Safeguarding standards':                 'Met' (1319), 'Not met' (10)
+#   'Inclusion':                               'Expected standard' (710),
+#       'Strong standard' (447), 'Needs attention' (130), 'Exceptional' (23),
+#       'Urgent improvement' (19)
+#   'Curriculum and teaching':                 'Expected standard' (797),
+#       'Needs attention' (287), 'Strong standard' (206),
+#       'Urgent improvement' (28), 'Exceptional' (11)
+#   'Achievement':                             'Expected standard' (701),
+#       'Needs attention' (364), 'Strong standard' (207),
+#       'Urgent improvement' (39), 'Exceptional' (18)
+#   'Attendance and behaviour':                'Expected standard' (699),
+#       'Strong standard' (405), 'Needs attention' (188),
+#       'Urgent improvement' (21), 'Exceptional' (16)
+#   'Personal development and wellbeing':      'Expected standard' (728),
+#       'Strong standard' (504), 'Needs attention' (66), 'Exceptional' (23),
+#       'Urgent improvement' (8)
+#   'Leadership and governance':               'Expected standard' (813),
+#       'Strong standard' (292), 'Needs attention' (172),
+#       'Urgent improvement' (34), 'Exceptional' (18)
+#   'Latest OEIF  safeguarding is effective?' (note double space, not used by
+#   Task 7 -- kept for completeness): 'Yes' (12970), 'No' (96)
+#
+# So the 6 graded report-card columns share exactly one 5-value vocabulary:
+# {'Exceptional', 'Strong standard', 'Expected standard', 'Needs attention',
+# 'Urgent improvement'} -- no 'Attention needed' variant was observed
+# anywhere, so parse_report_card_grade.sql does NOT need that speculative
+# branch from the task brief. 'Safeguarding standards' is a separate
+# two-value vocabulary {'Met', 'Not met'}.
+#
+# Collision check: 'Achievement' matches by EXACT list-membership
+# (`candidate in df_columns`, a Python list containment check against the
+# full column-name list, not a substring/regex match) against only
+# ['Achievement', 'Achievement - date of grade'] -- the date-paired column
+# has a different exact string and is never selected. Same check for
+# 'Safeguarding standards' found only itself, its own date-of-grade column,
+# and the unrelated 'Latest OEIF  safeguarding is effective?' column (not
+# mapped to any rc_* field). No legacy OEIF column is accidentally consumed
+# by an rc_ mapping.

pipeline/transform/macros/parse_report_card_grade.sql

@@ -0,0 +1,17 @@
+-- Macro: Parse Ofsted Report Card grade (post-Nov 2025 framework) from text
+-- into the 5-point scale. Real values confirmed via a live sample of the MI
+-- CSV (see pipeline/scripts/diagnose_compare_gaps.py's
+-- "TASK 7 VALUE SAMPLE 2026-07-12" note) -- unrecognised text (including the
+-- 'NULL' sentinel used by the source CSV for blanks) parses to NULL, never
+-- errors.
+
+{% macro parse_report_card_grade(column_name) %}
+    case lower(trim(nullif({{ column_name }}, 'NULL')))
+        when 'exceptional' then 1
+        when 'strong standard' then 2
+        when 'expected standard' then 3
+        when 'needs attention' then 4
+        when 'urgent improvement' then 5
+        else null
+    end
+{% endmacro %}

pipeline/transform/models/intermediate/int_ks2_with_lineage.sql

@@ -15,8 +15,11 @@ current_ks2 as (
         year, total_pupils, eligible_pupils,
         rwm_expected_pct, rwm_high_pct,
         reading_expected_pct, reading_high_pct, reading_avg_score, reading_progress,
+        reading_progress_lower_ci, reading_progress_upper_ci,
         writing_expected_pct, writing_high_pct, writing_progress,
+        writing_progress_lower_ci, writing_progress_upper_ci, writing_working_towards_pct,
         maths_expected_pct, maths_high_pct, maths_avg_score, maths_progress,
+        maths_progress_lower_ci, maths_progress_upper_ci,
         gps_expected_pct, gps_high_pct, gps_avg_score, science_expected_pct,
         reading_absence_pct, writing_absence_pct, maths_absence_pct, gps_absence_pct, science_absence_pct,
         rwm_expected_boys_pct, rwm_high_boys_pct, rwm_expected_girls_pct, rwm_high_girls_pct,
@@ -33,8 +36,11 @@ predecessor_ks2 as (
         ks2.year, ks2.total_pupils, ks2.eligible_pupils,
         ks2.rwm_expected_pct, ks2.rwm_high_pct,
         ks2.reading_expected_pct, ks2.reading_high_pct, ks2.reading_avg_score, ks2.reading_progress,
+        ks2.reading_progress_lower_ci, ks2.reading_progress_upper_ci,
         ks2.writing_expected_pct, ks2.writing_high_pct, ks2.writing_progress,
+        ks2.writing_progress_lower_ci, ks2.writing_progress_upper_ci, ks2.writing_working_towards_pct,
         ks2.maths_expected_pct, ks2.maths_high_pct, ks2.maths_avg_score, ks2.maths_progress,
+        ks2.maths_progress_lower_ci, ks2.maths_progress_upper_ci,
         ks2.gps_expected_pct, ks2.gps_high_pct, ks2.gps_avg_score, ks2.science_expected_pct,
         ks2.reading_absence_pct, ks2.writing_absence_pct, ks2.maths_absence_pct, ks2.gps_absence_pct, ks2.science_absence_pct,
         ks2.rwm_expected_boys_pct, ks2.rwm_high_boys_pct, ks2.rwm_expected_girls_pct, ks2.rwm_high_girls_pct,

pipeline/transform/models/intermediate/int_ks4_with_lineage.sql

@@ -18,7 +18,8 @@ current_ks4 as (
         english_maths_strong_pass_pct, english_maths_standard_pass_pct,
         ebacc_entry_pct, ebacc_strong_pass_pct, ebacc_standard_pass_pct, ebacc_avg_score,
         gcse_grade_91_pct,
-        sen_pct, sen_support_pct, sen_ehcp_pct
+        sen_pct, sen_support_pct, sen_ehcp_pct,
+        progress_8_banding, attainment_8_disadvantage_gap, progress_8_disadvantage_gap
     from all_ks4
 ),
 
@@ -34,7 +35,8 @@ predecessor_ks4 as (
         ks4.english_maths_strong_pass_pct, ks4.english_maths_standard_pass_pct,
         ks4.ebacc_entry_pct, ks4.ebacc_strong_pass_pct, ks4.ebacc_standard_pass_pct, ks4.ebacc_avg_score,
         ks4.gcse_grade_91_pct,
-        ks4.sen_pct, ks4.sen_support_pct, ks4.sen_ehcp_pct
+        ks4.sen_pct, ks4.sen_support_pct, ks4.sen_ehcp_pct,
+        ks4.progress_8_banding, ks4.attainment_8_disadvantage_gap, ks4.progress_8_disadvantage_gap
     from all_ks4 ks4
     inner join {{ ref('int_school_lineage') }} lin
         on ks4.urn = lin.predecessor_urn

pipeline/transform/models/marts/_marts_schema.yml

@@ -86,6 +86,13 @@ models:
         tests: [not_null]
       - name: year
         tests: [not_null]
+      - name: reading_progress_lower_ci
+      - name: reading_progress_upper_ci
+      - name: writing_progress_lower_ci
+      - name: writing_progress_upper_ci
+      - name: writing_working_towards_pct
+      - name: maths_progress_lower_ci
+      - name: maths_progress_upper_ci
     tests:
       - unique:
           column_name: "urn || '-' || year"
@@ -97,6 +104,15 @@ models:
         tests: [not_null]
       - name: year
         tests: [not_null]
+      - name: progress_8_banding
+        tests:
+          - accepted_values:
+              values: ['Well above average', 'Above average', 'Average', 'Below average', 'Well below average']
+              config:
+                where: "progress_8_banding is not null"
+                severity: warn
+      - name: attainment_8_disadvantage_gap
+      - name: progress_8_disadvantage_gap
     tests:
       - unique:
           column_name: "urn || '-' || year"

pipeline/transform/models/marts/fact_ks2_performance.sql

@@ -15,13 +15,20 @@ select
     reading_high_pct,
     reading_avg_score,
     reading_progress,
+    reading_progress_lower_ci,
+    reading_progress_upper_ci,
     writing_expected_pct,
     writing_high_pct,
     writing_progress,
+    writing_progress_lower_ci,
+    writing_progress_upper_ci,
+    writing_working_towards_pct,
     maths_expected_pct,
     maths_high_pct,
     maths_avg_score,
     maths_progress,
+    maths_progress_lower_ci,
+    maths_progress_upper_ci,
     gps_expected_pct,
     gps_high_pct,
     gps_avg_score,

pipeline/transform/models/marts/fact_ks4_performance.sql

@@ -16,6 +16,9 @@ select
     progress_8_score,
     progress_8_lower_ci,
     progress_8_upper_ci,
+    progress_8_banding,
+    attainment_8_disadvantage_gap,
+    progress_8_disadvantage_gap,
     progress_8_english,
     progress_8_maths,
     progress_8_ebacc,

pipeline/transform/models/staging/stg_ees_ks2.sql

@@ -39,6 +39,12 @@ pivoted as (
         max(case when subject = 'Reading'
                   and breakdown_topic = 'All pupils' and breakdown = 'Total'
             then {{ safe_numeric('progress_measure_score') }} end) as reading_progress,
+        max(case when subject = 'Reading'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as reading_progress_lower_ci,
+        max(case when subject = 'Reading'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as reading_progress_upper_ci,
         max(case when subject = 'Reading'
                   and breakdown_topic = 'All pupils' and breakdown = 'Total'
             then {{ safe_numeric('absent_or_not_able_to_access_percent') }} end) as reading_absence_pct,
@@ -53,6 +59,15 @@ pivoted as (
         max(case when subject = 'Writing'
                   and breakdown_topic = 'All pupils' and breakdown = 'Total'
             then {{ safe_numeric('progress_measure_score') }} end) as writing_progress,
+        max(case when subject = 'Writing'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as writing_progress_lower_ci,
+        max(case when subject = 'Writing'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as writing_progress_upper_ci,
+        max(case when subject = 'Writing'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('working_towards_expected_standard_pupil_percent') }} end) as writing_working_towards_pct,
         max(case when subject = 'Writing'
                   and breakdown_topic = 'All pupils' and breakdown = 'Total'
             then {{ safe_numeric('absent_or_not_able_to_access_percent') }} end) as writing_absence_pct,
@@ -70,6 +85,12 @@ pivoted as (
         max(case when subject = 'Maths'
                   and breakdown_topic = 'All pupils' and breakdown = 'Total'
             then {{ safe_numeric('progress_measure_score') }} end) as maths_progress,
+        max(case when subject = 'Maths'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('progress_measure_lower_conf_interval') }} end) as maths_progress_lower_ci,
+        max(case when subject = 'Maths'
+                  and breakdown_topic = 'All pupils' and breakdown = 'Total'
+            then {{ safe_numeric('progress_measure_upper_conf_interval') }} end) as maths_progress_upper_ci,
         max(case when subject = 'Maths'
                   and breakdown_topic = 'All pupils' and breakdown = 'Total'
             then {{ safe_numeric('absent_or_not_able_to_access_percent') }} end) as maths_absence_pct,
@@ -143,13 +164,20 @@ select
     p.reading_high_pct,
     p.reading_avg_score,
     p.reading_progress,
+    p.reading_progress_lower_ci,
+    p.reading_progress_upper_ci,
     p.writing_expected_pct,
     p.writing_high_pct,
     p.writing_progress,
+    p.writing_progress_lower_ci,
+    p.writing_progress_upper_ci,
+    p.writing_working_towards_pct,
     p.maths_expected_pct,
     p.maths_high_pct,
     p.maths_avg_score,
     p.maths_progress,
+    p.maths_progress_lower_ci,
+    p.maths_progress_upper_ci,
     p.gps_expected_pct,
     p.gps_high_pct,
     p.gps_avg_score,

pipeline/transform/models/staging/stg_ees_ks2_national.sql

@@ -31,4 +31,10 @@ select
 
 from {{ source('raw', 'ees_ks2_national') }}
 where time_period ~ '^[0-9]+$'
-  and cast(trim(time_period) as integer) >= 201617
+  -- 2015/16 was the first year of the current expected-standard tests, so it's
+  -- the correct floor (not 2016/17 -- that excluded a real, comparable national
+  -- row). GPS/science/scaled-score columns are already mapped correctly end to
+  -- end (tap.py's _KS2_NATIONAL_COL_MAP + this model select them fine); the
+  -- prod NULLs for those fields are stale raw.ees_ks2_national data from before
+  -- the map covered them, not a mapping bug -- no map change accompanies this fix.
+  and cast(trim(time_period) as integer) >= 201516

pipeline/transform/models/staging/stg_ees_ks4.sql

@@ -62,7 +62,16 @@ info as (
         {{ safe_numeric('ks2_scaledscore_average') }} as prior_attainment_avg,
         {{ safe_numeric('sen_pupil_percent') }} as sen_pct,
         {{ safe_numeric('sen_with_ehcp_pupil_percent') }} as sen_ehcp_pct,
-        {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct
+        {{ safe_numeric('sen_no_ehcp_pupil_percent') }} as sen_support_pct,
+        -- EES suppression sentinels (z/c/x/q/u) and blanks must not reach the
+        -- mart as banding labels
+        case
+            when lower(trim(progress8_banding)) in ('', 'z', 'c', 'x', 'q', 'u', 'null')
+                then null
+            else trim(progress8_banding)
+        end as progress_8_banding,
+        {{ safe_numeric('attainment8_diffn') }} as attainment_8_disadvantage_gap,
+        {{ safe_numeric('progress8_diffn') }} as progress_8_disadvantage_gap
     from {{ source('raw', 'ees_ks4_info') }}
     where school_urn is not null
 )
@@ -102,7 +111,10 @@ select
     -- Context
     i.sen_pct,
     i.sen_ehcp_pct,
-    i.sen_support_pct
+    i.sen_support_pct,
+    i.progress_8_banding,
+    i.attainment_8_disadvantage_gap,
+    i.progress_8_disadvantage_gap
 
 from all_pupils p
 left join info i on p.urn = i.urn and p.year = i.year

pipeline/transform/models/staging/stg_legacy_ks2.sql

@@ -17,13 +17,23 @@ select
     {{ safe_numeric('reading_high_pct') }} as reading_high_pct,
     {{ safe_numeric('reading_avg_score') }} as reading_avg_score,
     {{ safe_numeric('reading_progress') }} as reading_progress,
+    -- Progress CIs / working-towards: not published in the legacy CSVs.
+    -- Typed placeholders keep positional alignment with stg_ees_ks2 in
+    -- int_ks2_with_lineage's UNION ALL.
+    null::numeric as reading_progress_lower_ci,
+    null::numeric as reading_progress_upper_ci,
     {{ safe_numeric('writing_expected_pct') }} as writing_expected_pct,
     {{ safe_numeric('writing_high_pct') }} as writing_high_pct,
     {{ safe_numeric('writing_progress') }} as writing_progress,
+    null::numeric as writing_progress_lower_ci,
+    null::numeric as writing_progress_upper_ci,
+    null::numeric as writing_working_towards_pct,
     {{ safe_numeric('maths_expected_pct') }} as maths_expected_pct,
     {{ safe_numeric('maths_high_pct') }} as maths_high_pct,
     {{ safe_numeric('maths_avg_score') }} as maths_avg_score,
     {{ safe_numeric('maths_progress') }} as maths_progress,
+    null::numeric as maths_progress_lower_ci,
+    null::numeric as maths_progress_upper_ci,
     {{ safe_numeric('gps_expected_pct') }} as gps_expected_pct,
     {{ safe_numeric('gps_high_pct') }} as gps_high_pct,
     {{ safe_numeric('gps_avg_score') }} as gps_avg_score,

pipeline/transform/models/staging/stg_legacy_ks4.sql

@@ -41,8 +41,13 @@ select
 
     -- SEN
     null::numeric                           as sen_pct,
+    {{ safe_numeric('sen_ehcp_pct') }}      as sen_ehcp_pct,
     {{ safe_numeric('sen_support_pct') }}   as sen_support_pct,
-    {{ safe_numeric('sen_ehcp_pct') }}      as sen_ehcp_pct
+
+    -- Progress 8 banding & disadvantage gaps (not published in legacy format)
+    null::text    as progress_8_banding,
+    null::numeric as attainment_8_disadvantage_gap,
+    null::numeric as progress_8_disadvantage_gap
 
 from {{ source('raw', 'legacy_ks4') }}
 where urn is not null

pipeline/transform/models/staging/stg_ofsted_inspections.sql

@@ -33,17 +33,23 @@ renamed as (
         nullif(trim(ungraded_outcome), 'NULL')                                  as ungraded_outcome,
         {{ parse_ungraded_outcome('ungraded_outcome') }}::integer               as ungraded_grade,
 
-        -- Report Card fields (post-Nov 2025 framework)
+        -- Report Card fields (post-Nov 2025 framework), 5-point scale:
-        -- TODO: add rc_* columns to tap-uk-ofsted schema once CSV column names are confirmed
+        -- 1 Exceptional · 2 Strong standard · 3 Expected standard
-        null::text as rc_safeguarding_met,
+        -- · 4 Needs attention · 5 Urgent improvement
-        null::text as rc_inclusion,
+        case lower(trim(nullif(rc_safeguarding_met, 'NULL')))
-        null::text as rc_curriculum_teaching,
+            when 'met' then true
-        null::text as rc_achievement,
+            when 'not met' then false
-        null::text as rc_attendance_behaviour,
+        end                                                               as rc_safeguarding_met,
-        null::text as rc_personal_development,
+        {{ parse_report_card_grade('rc_inclusion') }}::integer            as rc_inclusion,
-        null::text as rc_leadership_governance,
+        {{ parse_report_card_grade('rc_curriculum_teaching') }}::integer  as rc_curriculum_teaching,
-        null::text as rc_early_years,
+        {{ parse_report_card_grade('rc_achievement') }}::integer          as rc_achievement,
-        null::text as rc_sixth_form,
+        {{ parse_report_card_grade('rc_attendance_behaviour') }}::integer as rc_attendance_behaviour,
+        {{ parse_report_card_grade('rc_personal_development') }}::integer as rc_personal_development,
+        {{ parse_report_card_grade('rc_leadership_governance') }}::integer as rc_leadership_governance,
+        -- No MI column exists for these yet (see tap.py); the tap never
+        -- emits rc_early_years/rc_sixth_form, so these stay NULL.
+        null::integer as rc_early_years,
+        null::integer as rc_sixth_form,
 
         report_url
     from source