From 6cc135ec5824f6cb829adf725e1057c49cf5aa60 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 13 Oct 2026 21:26:05 +0100 Subject: [PATCH] feat(dbt): further education status where a school has no school record (H4) Co-Authored-By: Claude Opus 5.5 --- .../models/intermediate/int_ofsted_latest.sql | 107 ++++++++++-------- .../models/intermediate/int_ofsted_latest.yml | 46 ++++++++ .../transform/models/marts/_marts_schema.yml | 2 +- ...ssert_ofsted_fe_never_overrides_school.sql | 7 ++ 4 files changed, 115 insertions(+), 47 deletions(-) create mode 100644 pipeline/transform/tests/assert_ofsted_fe_never_overrides_school.sql diff --git a/pipeline/transform/models/intermediate/int_ofsted_latest.sql b/pipeline/transform/models/intermediate/int_ofsted_latest.sql index 9dde247..dc41c5d 100644 --- a/pipeline/transform/models/intermediate/int_ofsted_latest.sql +++ b/pipeline/transform/models/intermediate/int_ofsted_latest.sql @@ -3,6 +3,8 @@ -- inspection) and the overall grade still in force, if any. A grade is dated -- by the inspection that awarded or confirmed it, never by a later visit. -- Rule and examples: docs/superpowers/specs/2026-10-05-ofsted-current-status-design.md +-- Schools with no school-framework record take their further education and +-- skills status from int_ofsted_fe_latest (same columns, same order; audit H4). with inspections as ( select @@ -62,51 +64,64 @@ graded as ( then 'graded_earlier' end as grade_case from latest +), + +school_status as ( + select + urn, + latest_visit_date, + latest_visit_kind, + case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome, + case grade_case + when 'confirmed' then ungraded_grade + when 'graded_latest' then overall_effectiveness + when 'graded_earlier' then overall_effectiveness + end as current_grade, + case grade_case + when 'confirmed' then ungraded_inspection_date + when 'graded_latest' then graded_inspection_date + when 'graded_earlier' then graded_inspection_date + end as current_grade_date, + case grade_case + when 'confirmed' then 'confirmed' + when 'graded_latest' then 'graded' + when 'graded_earlier' then 'graded' + end as current_grade_basis, + graded_inspection_date, + ungraded_inspection_date, + inspection_date, + inspection_type, + framework, + overall_effectiveness, + quality_of_education, + behaviour_attitudes, + personal_development, + leadership_management, + early_years_provision, + sixth_form_provision, + ungraded_outcome, + ungraded_grade, + rc_safeguarding_met, + rc_inclusion, + rc_curriculum_teaching, + rc_achievement, + rc_attendance_behaviour, + rc_personal_development, + rc_leadership_governance, + rc_early_years, + rc_sixth_form, + rc_inspection_date, + report_url + from graded +), + +fe_status as ( + -- Same columns in the same order as school_status. + select f.* + from {{ ref('int_ofsted_fe_latest') }} f + where not exists (select 1 from school_status s where s.urn = f.urn) ) -select - urn, - latest_visit_date, - latest_visit_kind, - case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome, - case grade_case - when 'confirmed' then ungraded_grade - when 'graded_latest' then overall_effectiveness - when 'graded_earlier' then overall_effectiveness - end as current_grade, - case grade_case - when 'confirmed' then ungraded_inspection_date - when 'graded_latest' then graded_inspection_date - when 'graded_earlier' then graded_inspection_date - end as current_grade_date, - case grade_case - when 'confirmed' then 'confirmed' - when 'graded_latest' then 'graded' - when 'graded_earlier' then 'graded' - end as current_grade_basis, - graded_inspection_date, - ungraded_inspection_date, - inspection_date, - inspection_type, - framework, - overall_effectiveness, - quality_of_education, - behaviour_attitudes, - personal_development, - leadership_management, - early_years_provision, - sixth_form_provision, - ungraded_outcome, - ungraded_grade, - rc_safeguarding_met, - rc_inclusion, - rc_curriculum_teaching, - rc_achievement, - rc_attendance_behaviour, - rc_personal_development, - rc_leadership_governance, - rc_early_years, - rc_sixth_form, - rc_inspection_date, - report_url -from graded +select * from school_status +union all +select * from fe_status diff --git a/pipeline/transform/models/intermediate/int_ofsted_latest.yml b/pipeline/transform/models/intermediate/int_ofsted_latest.yml index da5ae0f..b79c9cc 100644 --- a/pipeline/transform/models/intermediate/int_ofsted_latest.yml +++ b/pipeline/transform/models/intermediate/int_ofsted_latest.yml @@ -8,6 +8,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 102408, graded_inspection_date: '2025-06-17', ungraded_inspection_date: '2020-02-06', overall_effectiveness: null, ungraded_grade: 2, ungraded_outcome: 'School remains Good'} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 102408, latest_visit_date: '2025-06-17', latest_visit_kind: graded, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null} @@ -18,6 +20,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2019-06-01', current_grade_basis: graded} @@ -29,6 +33,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 104762, graded_inspection_date: '2020-01-07', ungraded_inspection_date: '2024-07-18', overall_effectiveness: 2, ungraded_grade: 2, ungraded_outcome: 'School remains Good'} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 104762, latest_visit_date: '2024-07-18', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: 2, current_grade_date: '2024-07-18', current_grade_basis: confirmed} @@ -40,6 +46,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 139888, graded_inspection_date: '2020-03-03', ungraded_inspection_date: '2025-05-21', overall_effectiveness: 2, ungraded_grade: null, ungraded_outcome: 'Standards maintained'} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 139888, latest_visit_date: '2025-05-21', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: 2, current_grade_date: '2020-03-03', current_grade_basis: graded} @@ -51,6 +59,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 136454, ungraded_inspection_date: '2024-11-13', ungraded_grade: null, ungraded_outcome: 'Standards maintained'} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 136454, latest_visit_date: '2024-11-13', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: null, current_grade_date: null, current_grade_basis: null} @@ -62,6 +72,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 110048, ungraded_inspection_date: '2023-10-05', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2026-05-06', rc_inclusion: 3} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 110048, latest_visit_date: '2026-05-06', latest_visit_kind: report_card, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null} @@ -74,6 +86,8 @@ unit_tests: rows: - {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3} - {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3, rc_inspection_date: '2026-06-02', rc_inclusion: 3} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 138186, latest_visit_date: '2026-06-02', latest_visit_kind: report_card, current_grade: null} @@ -84,6 +98,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 2, graded_inspection_date: '2024-03-01', ungraded_inspection_date: '2024-03-01', overall_effectiveness: 1, ungraded_grade: 2, ungraded_outcome: 'School remains Good'} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 2, latest_visit_kind: graded, current_grade: 1, current_grade_basis: graded} @@ -94,6 +110,8 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 3, graded_inspection_date: '2018-05-01', overall_effectiveness: 9} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 3, latest_visit_kind: graded, current_grade: null} @@ -108,6 +126,34 @@ unit_tests: - input: ref('stg_ofsted_inspections') rows: - {urn: 4, ungraded_inspection_date: '2026-03-02', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2025-12-01', rc_inclusion: 3} + - input: ref('int_ofsted_fe_latest') + rows: [] expect: rows: - {urn: 4, latest_visit_date: '2026-03-02', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: null, current_grade_date: null, current_grade_basis: null} + + - name: fe_status_fills_a_school_with_no_school_record + description: Queen Mary's College (145175) has no school-framework record; its FE status passes through (audit H4). + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: [] + - input: ref('int_ofsted_fe_latest') + rows: + - {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, framework: 'Further education and skills'} + expect: + rows: + - {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, framework: 'Further education and skills'} + + - name: a_school_record_wins_over_an_fe_record + model: int_ofsted_latest + given: + - input: ref('stg_ofsted_inspections') + rows: + - {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2, framework: 'Schools - S5'} + - input: ref('int_ofsted_fe_latest') + rows: + - {urn: 1, latest_visit_date: '2026-05-12', latest_visit_kind: report_card, rc_inspection_date: '2026-05-12', framework: 'Further education and skills'} + expect: + rows: + - {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, framework: 'Schools - S5'} diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 4355775..4d36b92 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -138,7 +138,7 @@ models: tests: - not_null - accepted_values: - values: ['report_card', 'graded', 'ungraded'] + values: ['report_card', 'graded', 'ungraded', 'short'] - name: current_grade tests: - accepted_values: diff --git a/pipeline/transform/tests/assert_ofsted_fe_never_overrides_school.sql b/pipeline/transform/tests/assert_ofsted_fe_never_overrides_school.sql new file mode 100644 index 0000000..61df256 --- /dev/null +++ b/pipeline/transform/tests/assert_ofsted_fe_never_overrides_school.sql @@ -0,0 +1,7 @@ +-- A school-framework record always wins over a further education one for the +-- same URN (audit H4): no FE row may stand where Ofsted's school MI has one. + +select f.urn +from {{ ref('fact_ofsted_latest') }} f +join {{ ref('stg_ofsted_inspections') }} s on s.urn = f.urn +where f.framework = 'Further education and skills'