diff --git a/pipeline/transform/models/intermediate/int_ofsted_fe_latest.sql b/pipeline/transform/models/intermediate/int_ofsted_fe_latest.sql new file mode 100644 index 0000000..9b292fa --- /dev/null +++ b/pipeline/transform/models/intermediate/int_ofsted_fe_latest.sql @@ -0,0 +1,98 @@ +-- Intermediate model: Ofsted further education and skills status per URN +-- (audit H4), in int_ofsted_latest's columns so it can be unioned there. +-- A provider matches a school by Provider URN, else by UKPRN (stg_gias_ukprn). +-- Rule, in the C1 status rule's terms +-- (docs/superpowers/specs/2026-10-06-inspection-status-beyond-ofsted-schools-design.md): +-- a report card on or after the previous-framework inspection wins and +-- leaves no grade in force; otherwise a previous-framework grade 1-4 is the +-- current grade, dated by that inspection; the latest visit is the later of +-- it and any short inspection (whose outcome the MI does not give). + +with gias as ( + select urn, ukprn from {{ ref('stg_gias_ukprn') }} +), + +providers as ( + select distinct on (g.urn) + g.urn, + p.graded_inspection_date, + p.overall_effectiveness, + p.short_inspection_date + from gias g + join {{ ref('stg_ofsted_fe_providers') }} p + on p.provider_urn = g.urn + or (g.ukprn is not null and p.provider_ukprn = g.ukprn) + order by g.urn, (p.provider_urn = g.urn) desc, p.graded_inspection_date desc nulls last +), + +report_cards as ( + select distinct on (g.urn) + g.urn, + c.rc_inspection_date, + c.rc_safeguarding_met + from gias g + join {{ ref('stg_ofsted_fe_inspections') }} c + on c.provider_urn = g.urn + or (g.ukprn is not null and c.provider_ukprn = g.ukprn) + order by g.urn, (c.provider_urn = g.urn) desc, c.rc_inspection_date desc, c.inspection_number desc +), + +combined as ( + select + coalesce(p.urn, c.urn) as urn, + p.graded_inspection_date, + p.overall_effectiveness, + p.short_inspection_date, + c.rc_inspection_date as card_date, + c.rc_safeguarding_met as card_safeguarding_met + from providers p + full outer join report_cards c on c.urn = p.urn +), + +decided as ( + select + *, + card_date is not null + and card_date >= coalesce(greatest(graded_inspection_date, short_inspection_date), card_date) + as report_card_latest + from combined +) + +select + urn, + case when report_card_latest then card_date + else greatest(graded_inspection_date, short_inspection_date) end as latest_visit_date, + case when report_card_latest then 'report_card' + when short_inspection_date > graded_inspection_date then 'short' + else 'graded' end as latest_visit_kind, + null::text as latest_visit_outcome, + case when not report_card_latest then overall_effectiveness end as current_grade, + case when not report_card_latest then graded_inspection_date end as current_grade_date, + case when not report_card_latest then 'graded' end as current_grade_basis, + graded_inspection_date, + null::date as ungraded_inspection_date, + coalesce(graded_inspection_date, card_date) as inspection_date, + null::text as inspection_type, + 'Further education and skills'::text as framework, + overall_effectiveness, + null::integer as quality_of_education, + null::integer as behaviour_attitudes, + null::integer as personal_development, + null::integer as leadership_management, + null::integer as early_years_provision, + null::integer as sixth_form_provision, + null::text as ungraded_outcome, + null::integer as ungraded_grade, + case when report_card_latest then card_safeguarding_met end as rc_safeguarding_met, + null::integer as rc_inclusion, + null::integer as rc_curriculum_teaching, + null::integer as rc_achievement, + null::integer as rc_attendance_behaviour, + null::integer as rc_personal_development, + null::integer as rc_leadership_governance, + null::integer as rc_early_years, + null::integer as rc_sixth_form, + case when report_card_latest then card_date end as rc_inspection_date, + null::text as report_url +from decided +where report_card_latest or overall_effectiveness between 1 and 4 diff --git a/pipeline/transform/models/intermediate/int_ofsted_fe_latest.yml b/pipeline/transform/models/intermediate/int_ofsted_fe_latest.yml new file mode 100644 index 0000000..e4c1080 --- /dev/null +++ b/pipeline/transform/models/intermediate/int_ofsted_fe_latest.yml @@ -0,0 +1,128 @@ +version: 2 + +unit_tests: + - name: int_ofsted_fe_grade_only + description: Queen Mary's College (145175), Good on 6 December 2022, no report card. + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 145175, ukprn: '10005429'} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 145175, provider_ukprn: '10005429', graded_inspection_date: '2022-12-06', overall_effectiveness: 2} + - input: ref('stg_ofsted_fe_inspections') + rows: [] + expect: + rows: + - {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, rc_inspection_date: null, framework: 'Further education and skills'} + + - name: int_ofsted_fe_short_inspection_is_a_visit_not_a_grade + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 1, ukprn: null} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 1, graded_inspection_date: '2019-03-05', overall_effectiveness: 2, short_inspection_date: '2023-02-01'} + - input: ref('stg_ofsted_fe_inspections') + rows: [] + expect: + rows: + - {urn: 1, latest_visit_date: '2023-02-01', latest_visit_kind: short, current_grade: 2, current_grade_date: '2019-03-05', current_grade_basis: graded} + + - name: int_ofsted_fe_newer_report_card_wins + description: The Autism Project - CareTrade (143526). Good 2021; report card 12 May 2026, safeguarding met. + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 143526, ukprn: '10000002'} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 143526, provider_ukprn: '10000002', graded_inspection_date: '2021-11-16', overall_effectiveness: 2} + - input: ref('stg_ofsted_fe_inspections') + rows: + - {inspection_number: '10380001', provider_urn: 143526, provider_ukprn: '10000002', rc_inspection_date: '2026-05-12', rc_safeguarding_met: true} + expect: + rows: + - {urn: 143526, latest_visit_date: '2026-05-12', latest_visit_kind: report_card, current_grade: null, current_grade_date: null, current_grade_basis: null, rc_inspection_date: '2026-05-12', rc_safeguarding_met: true, overall_effectiveness: 2} + + - name: int_ofsted_fe_report_card_only + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 150042, ukprn: null} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 150042, graded_inspection_date: null, overall_effectiveness: null} + - input: ref('stg_ofsted_fe_inspections') + rows: + - {inspection_number: '10380002', provider_urn: 150042, rc_inspection_date: '2026-01-20', rc_safeguarding_met: false} + expect: + rows: + - {urn: 150042, latest_visit_date: '2026-01-20', latest_visit_kind: report_card, current_grade: null, rc_inspection_date: '2026-01-20', rc_safeguarding_met: false} + + - name: int_ofsted_fe_listed_never_inspected_has_no_row + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 130645, ukprn: null} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 130645, graded_inspection_date: null, overall_effectiveness: null} + - input: ref('stg_ofsted_fe_inspections') + rows: [] + expect: + rows: [] + + - name: int_ofsted_fe_matches_by_ukprn_when_the_urn_differs + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 5, ukprn: '10099999'} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 9999999, provider_ukprn: '10099999', graded_inspection_date: '2024-01-30', overall_effectiveness: 3} + - input: ref('stg_ofsted_fe_inspections') + rows: [] + expect: + rows: + - {urn: 5, current_grade: 3, current_grade_date: '2024-01-30'} + + - name: int_ofsted_fe_prefers_the_urn_match + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 6, ukprn: '10088888'} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 6, provider_ukprn: '10077777', graded_inspection_date: '2023-05-01', overall_effectiveness: 1} + - {provider_urn: 8888888, provider_ukprn: '10088888', graded_inspection_date: '2024-05-01', overall_effectiveness: 4} + - input: ref('stg_ofsted_fe_inspections') + rows: [] + expect: + rows: + - {urn: 6, current_grade: 1, current_grade_date: '2023-05-01'} + + - name: int_ofsted_fe_report_card_older_than_grade_is_ignored + description: Bad data guard. A report card dated before the graded inspection cannot be the latest, and a row must never carry both a grade and a report card. + model: int_ofsted_fe_latest + given: + - input: ref('stg_gias_ukprn') + rows: + - {urn: 7, ukprn: null} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 7, graded_inspection_date: '2026-03-01', overall_effectiveness: 2} + - input: ref('stg_ofsted_fe_inspections') + rows: + - {inspection_number: '1', provider_urn: 7, rc_inspection_date: '2026-01-10', rc_safeguarding_met: true} + expect: + rows: + - {urn: 7, latest_visit_kind: graded, current_grade: 2, rc_inspection_date: null, rc_safeguarding_met: null}