From e6babe21f5f56cb816a9bcaf3ab5f03aed06d6d7 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 5 Oct 2026 16:23:38 +0100 Subject: [PATCH] feat(pipeline): work out each school's current Ofsted status once int_ofsted_latest now picks each school's latest visit (report card, graded or ungraded inspection) and the overall grade still in force, dated by the inspection that awarded or confirmed it. A 'School remains Good' from an older ungraded visit is no longer carried past a newer inspection that gave no grade (audit C1). Duplicate monthly rows resolve to the newest visit, so a newer report card always wins. dim_school's columns are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../models/intermediate/int_ofsted_latest.sql | 83 ++++++++++++++-- .../models/intermediate/int_ofsted_latest.yml | 99 +++++++++++++++++++ 2 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 pipeline/transform/models/intermediate/int_ofsted_latest.yml diff --git a/pipeline/transform/models/intermediate/int_ofsted_latest.sql b/pipeline/transform/models/intermediate/int_ofsted_latest.sql index 2b09768..ac3727b 100644 --- a/pipeline/transform/models/intermediate/int_ofsted_latest.sql +++ b/pipeline/transform/models/intermediate/int_ofsted_latest.sql @@ -1,18 +1,88 @@ --- Intermediate model: Latest Ofsted inspection per URN --- Picks the most recent inspection for each school +-- Intermediate model: the current Ofsted status per URN +-- One row per school: its latest visit (report card, graded or ungraded +-- inspection) and the overall grade still in force, if any. A grade is dated +-- by the inspection that awarded or confirmed it, never by a later visit. +-- Rule and examples: docs/superpowers/specs/2026-10-05-ofsted-current-status-design.md -with ranked as ( +with inspections as ( select *, + -- Report cards began in Nov 2025, after the last legacy inspections, + -- so a school's report card is always its latest visit. + coalesce( + rc_inspection_date, + greatest(graded_inspection_date, ungraded_inspection_date) + ) as latest_visit_date + from {{ ref('stg_ofsted_inspections') }} +), + +ranked as ( + select + *, + -- Monthly loads can leave several rows per school. The newest visit + -- wins; the tie-breaks keep the choice deterministic. row_number() over ( partition by urn - order by inspection_date desc + order by latest_visit_date desc, + rc_inspection_date desc nulls last, + graded_inspection_date desc nulls last, + ungraded_inspection_date desc nulls last ) as rn - from {{ ref('stg_ofsted_inspections') }} + from inspections +), + +latest as ( + select + *, + case + when rc_inspection_date is not null then 'report_card' + when graded_inspection_date = latest_visit_date then 'graded' + else 'ungraded' + end as latest_visit_kind + from ranked + where rn = 1 +), + +graded as ( + select + *, + -- The grade still in force. A graded inspection's own overall grade + -- (1-4; "Not judged" and the sentinel 9 are no grade); else an + -- ungraded visit's "School remains X"; else, after an ungraded visit + -- that names no grade, the graded inspection's grade. + case + when latest_visit_kind = 'graded' and overall_effectiveness between 1 and 4 + then 'graded_latest' + when latest_visit_kind = 'ungraded' and ungraded_grade is not null + then 'confirmed' + when latest_visit_kind = 'ungraded' and overall_effectiveness between 1 and 4 + then 'graded_earlier' + end as grade_case + from latest ) select urn, + latest_visit_date, + latest_visit_kind, + case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome, + case grade_case + when 'confirmed' then ungraded_grade + when 'graded_latest' then overall_effectiveness + when 'graded_earlier' then overall_effectiveness + end as current_grade, + case grade_case + when 'confirmed' then ungraded_inspection_date + when 'graded_latest' then graded_inspection_date + when 'graded_earlier' then graded_inspection_date + end as current_grade_date, + case grade_case + when 'confirmed' then 'confirmed' + when 'graded_latest' then 'graded' + when 'graded_earlier' then 'graded' + end as current_grade_basis, + graded_inspection_date, + ungraded_inspection_date, inspection_date, inspection_type, framework, @@ -36,5 +106,4 @@ select rc_sixth_form, rc_inspection_date, report_url -from ranked -where rn = 1 +from graded diff --git a/pipeline/transform/models/intermediate/int_ofsted_latest.yml b/pipeline/transform/models/intermediate/int_ofsted_latest.yml new file mode 100644 index 0000000..c436b2e --- /dev/null +++ b/pipeline/transform/models/intermediate/int_ofsted_latest.yml @@ -0,0 +1,99 @@ +version: 2 + +unit_tests: + - name: graded_not_judged_has_no_grade + description: Rabbsfarm (102408). The 2025 inspection gave no overall grade, so the 2020 "remains Good" is not carried forward (audit C1). + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 102408, graded_inspection_date: '2025-06-17', ungraded_inspection_date: '2020-02-06', overall_effectiveness: null, ungraded_grade: 2, ungraded_outcome: 'School remains Good'} + expect: + rows: + - {urn: 102408, latest_visit_date: '2025-06-17', latest_visit_kind: graded, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null} + + - name: graded_with_overall_grade + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2} + expect: + rows: + - {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2019-06-01', current_grade_basis: graded} + + - name: ungraded_remains_good_confirms_the_grade + description: Robins Lane (104762). Graded Good 2020, "School remains Good" July 2024 — Good, dated by the confirming visit. + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 104762, graded_inspection_date: '2020-01-07', ungraded_inspection_date: '2024-07-18', overall_effectiveness: 2, ungraded_grade: 2, ungraded_outcome: 'School remains Good'} + expect: + rows: + - {urn: 104762, latest_visit_date: '2024-07-18', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: 2, current_grade_date: '2024-07-18', current_grade_basis: confirmed} + + - name: post_2024_ungraded_keeps_graded_grade_with_its_own_date + description: Washwood Heath (139888). Graded Good 2020, "Standards maintained" May 2025 — Good, dated 2020; latest visit May 2025 (audit M1). + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 139888, graded_inspection_date: '2020-03-03', ungraded_inspection_date: '2025-05-21', overall_effectiveness: 2, ungraded_grade: null, ungraded_outcome: 'Standards maintained'} + expect: + rows: + - {urn: 139888, latest_visit_date: '2025-05-21', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: 2, current_grade_date: '2020-03-03', current_grade_basis: graded} + + - name: ungraded_only_standards_maintained + description: Oakgrove (136454). Only an ungraded visit, outcome names no grade. + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 136454, ungraded_inspection_date: '2024-11-13', ungraded_grade: null, ungraded_outcome: 'Standards maintained'} + expect: + rows: + - {urn: 136454, latest_visit_date: '2024-11-13', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: null, current_grade_date: null, current_grade_basis: null} + + - name: report_card_wins + description: The Willink School (110048). A report card is the latest visit and no legacy grade stays in force. + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 110048, ungraded_inspection_date: '2023-10-05', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2026-05-06', rc_inclusion: 3} + expect: + rows: + - {urn: 110048, latest_visit_date: '2026-05-06', latest_visit_kind: report_card, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null} + + - name: duplicate_rows_newer_report_card_wins + description: Monthly loads can leave an older row beside a newer one for the same graded date; the row with the report card must win (audit H3). + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3} + - {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3, rc_inspection_date: '2026-06-02', rc_inclusion: 3} + expect: + rows: + - {urn: 138186, latest_visit_date: '2026-06-02', latest_visit_kind: report_card, current_grade: null} + + - name: same_day_graded_wins + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 2, graded_inspection_date: '2024-03-01', ungraded_inspection_date: '2024-03-01', overall_effectiveness: 1, ungraded_grade: 2, ungraded_outcome: 'School remains Good'} + expect: + rows: + - {urn: 2, latest_visit_kind: graded, current_grade: 1, current_grade_basis: graded} + + - name: overall_sentinel_is_not_a_grade + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 3, graded_inspection_date: '2018-05-01', overall_effectiveness: 9} + expect: + rows: + - {urn: 3, latest_visit_kind: graded, current_grade: null}