From 358705bf3934e555b609a2a29adc0a2f8a8e4249 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 5 Oct 2026 16:22:49 +0100 Subject: [PATCH] feat(pipeline): keep Ofsted's graded, ungraded and report-card dates apart int_ofsted_latest needs to know which inspection came last. Report-card-only schools are no longer dropped (123 schools, audit H3); inspection_date stays for current readers and now falls back to the report-card date, so fact_ofsted_inspection's not_null test still holds. Co-Authored-By: Claude Opus 5.5 --- .../models/staging/stg_ofsted_inspections.sql | 67 ++++++++++--------- .../models/staging/stg_ofsted_inspections.yml | 25 +++++++ 2 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 pipeline/transform/models/staging/stg_ofsted_inspections.yml diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_inspections.sql index 6bb7574..efdf118 100644 --- a/pipeline/transform/models/staging/stg_ofsted_inspections.sql +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.sql @@ -1,5 +1,10 @@ -- Staging model: Ofsted inspection records --- Handles both OEIF (pre-Nov 2025) and Report Card (post-Nov 2025) frameworks +-- Handles both OEIF (pre-Nov 2025) and Report Card (post-Nov 2025) frameworks. +-- +-- Ofsted's MI carries up to three inspections per school: the latest graded +-- one, the latest ungraded one and the latest report card. Their dates stay +-- separate here so int_ofsted_latest can tell which came last and which one a +-- grade belongs to (docs/superpowers/specs/2026-10-05-ofsted-current-status-design.md). with source as ( select * from {{ source('raw', 'ofsted_inspections') }} @@ -8,17 +13,12 @@ with source as ( renamed as ( select cast(urn as integer) as urn, - -- Inspection event date: the graded inspection when present, otherwise the - -- ungraded (Section 8) inspection so schools with only an ungraded - -- inspection are still retained. - coalesce( - to_date(nullif(trim(inspection_date), 'NULL'), 'DD/MM/YYYY'), - to_date(nullif(trim(ungraded_inspection_date), 'NULL'), 'DD/MM/YYYY') - ) as inspection_date, + to_date(nullif(trim(inspection_date), 'NULL'), 'DD/MM/YYYY') as graded_inspection_date, + to_date(nullif(trim(ungraded_inspection_date), 'NULL'), 'DD/MM/YYYY') as ungraded_inspection_date, inspection_type, event_type_grouping as framework, - -- OEIF grades (1-4 scale) + -- OEIF grades (1-4 scale; 9 = not applicable) {{ safe_numeric('overall_effectiveness') }}::integer as overall_effectiveness, {{ safe_numeric('quality_of_education') }}::integer as quality_of_education, {{ safe_numeric('behaviour_and_attitudes') }}::integer as behaviour_attitudes, @@ -27,9 +27,9 @@ renamed as ( {{ safe_numeric('early_years_provision') }}::integer as early_years_provision, {{ safe_numeric('sixth_form_provision') }}::integer as sixth_form_provision, - -- Ungraded (Section 8) inspection outcome — free text, plus a grade - -- parsed from it (1/2/null) used as a last-resort fallback for schools - -- with no graded overall effectiveness. + -- Ungraded (Section 8) inspection outcome — free text, plus the grade + -- it confirms ("School remains Good" → 2); null for outcomes that + -- name no grade. nullif(trim(ungraded_outcome), 'NULL') as ungraded_outcome, {{ parse_ungraded_outcome('ungraded_outcome') }}::integer as ungraded_grade, @@ -50,31 +50,36 @@ renamed as ( {{ parse_report_card_grade('rc_sixth_form') }}::integer as rc_sixth_form, -- Start date of the latest FULL inspection (the report-card - -- inspection in the renewed framework). Guarded in the final select: - -- only kept when the row actually carries report-card grades, because - -- in legacy-format files this column is the legacy inspection date. + -- inspection in the renewed framework). Only kept when the row + -- carries report-card grades, because in legacy-format files this + -- column is the legacy inspection date. to_date(nullif(trim(rc_inspection_date), 'NULL'), 'DD/MM/YYYY') as rc_inspection_date_raw, nullif(trim(report_url), 'NULL') as report_url from source where urn is not null - and ( - nullif(trim(inspection_date), 'NULL') is not null - or nullif(trim(ungraded_inspection_date), 'NULL') is not null - ) +), + +dated as ( + select + *, + case + when rc_safeguarding_met is not null + or rc_inclusion is not null + or rc_curriculum_teaching is not null + or rc_achievement is not null + or rc_attendance_behaviour is not null + or rc_personal_development is not null + or rc_leadership_governance is not null + then rc_inspection_date_raw + end as rc_inspection_date + from renamed ) select *, - case - when rc_safeguarding_met is not null - or rc_inclusion is not null - or rc_curriculum_teaching is not null - or rc_achievement is not null - or rc_attendance_behaviour is not null - or rc_personal_development is not null - or rc_leadership_governance is not null - then rc_inspection_date_raw - end as rc_inspection_date -from renamed -where inspection_date is not null + -- For readers that predate the separate dates (fact_ofsted_inspection and + -- the backend until it reads fact_ofsted_latest). Never null below. + coalesce(graded_inspection_date, ungraded_inspection_date, rc_inspection_date) as inspection_date +from dated +where coalesce(graded_inspection_date, ungraded_inspection_date, rc_inspection_date) is not null diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.yml b/pipeline/transform/models/staging/stg_ofsted_inspections.yml new file mode 100644 index 0000000..4b993d2 --- /dev/null +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.yml @@ -0,0 +1,25 @@ +version: 2 + +unit_tests: + - name: stg_ofsted_keeps_the_three_dates_apart + description: The graded and ungraded dates must stay separate so the latest visit can be found. + model: stg_ofsted_inspections + given: + - input: source('raw', 'ofsted_inspections') + rows: + - {urn: 102408, inspection_date: '17/06/2025', ungraded_inspection_date: '06/02/2020', overall_effectiveness: 'Not judged', ungraded_outcome: 'School remains Good', rc_inspection_date: '17/06/2025', rc_inclusion: 'NULL'} + expect: + rows: + - {urn: 102408, graded_inspection_date: '2025-06-17', ungraded_inspection_date: '2020-02-06', rc_inspection_date: null, inspection_date: '2025-06-17', ungraded_grade: 2} + + - name: stg_ofsted_keeps_report_card_only_rows + description: A school whose only inspection is a report card used to be dropped (audit H3). + model: stg_ofsted_inspections + given: + - input: source('raw', 'ofsted_inspections') + rows: + - {urn: 149612, inspection_date: 'NULL', ungraded_inspection_date: 'NULL', rc_inspection_date: '10/02/2026', rc_inclusion: 'Expected standard', rc_safeguarding_met: 'Met'} + - {urn: 1, inspection_date: 'NULL', ungraded_inspection_date: 'NULL', rc_inspection_date: 'NULL'} + expect: + rows: + - {urn: 149612, graded_inspection_date: null, ungraded_inspection_date: null, rc_inspection_date: '2026-02-10', inspection_date: '2026-02-10', rc_inclusion: 3}