diff --git a/pipeline/transform/models/staging/_stg_sources.yml b/pipeline/transform/models/staging/_stg_sources.yml index 7aa577f..1fc0a79 100644 --- a/pipeline/transform/models/staging/_stg_sources.yml +++ b/pipeline/transform/models/staging/_stg_sources.yml @@ -24,6 +24,12 @@ sources: - name: ofsted_inspections description: Ofsted Management Information inspection records + - name: ofsted_fe_providers + description: Ofsted further education and skills MI, newest "as at" file — one row per provider with its latest inspection under the previous framework + + - name: ofsted_fe_inspections + description: Ofsted further education and skills MI — renewed-framework inspections (report cards) since 10 November 2025, one row per inspection + - name: ees_ks2_attainment description: KS2 school attainment (long format — one row per school × subject × breakdown) diff --git a/pipeline/transform/models/staging/stg_ofsted_fe.yml b/pipeline/transform/models/staging/stg_ofsted_fe.yml new file mode 100644 index 0000000..713c7c6 --- /dev/null +++ b/pipeline/transform/models/staging/stg_ofsted_fe.yml @@ -0,0 +1,29 @@ +version: 2 + +unit_tests: + - name: stg_ofsted_fe_providers_parses_grades_and_dates + description: Ofsted writes "-" for no grade or date; grades are 1-4. + model: stg_ofsted_fe_providers + given: + - input: source('raw', 'ofsted_fe_providers') + rows: + - {provider_urn: '145175', provider_ukprn: '10005429', provider_name: "Queen Mary's College", renewed_eif_full_inspection: 'No', graded_inspection_date: '06/12/2022', overall_effectiveness: '2', short_inspection_date: '-'} + - {provider_urn: '130645', provider_ukprn: '-', provider_name: 'Exeter and North Devon Colleges Group', renewed_eif_full_inspection: 'No', graded_inspection_date: '-', overall_effectiveness: '-', short_inspection_date: '-'} + - {provider_urn: '', provider_ukprn: '10000001', provider_name: 'No URN'} + expect: + rows: + - {provider_urn: 145175, provider_ukprn: '10005429', renewed_eif_full_inspection: false, graded_inspection_date: '2022-12-06', overall_effectiveness: 2, short_inspection_date: null} + - {provider_urn: 130645, provider_ukprn: null, renewed_eif_full_inspection: false, graded_inspection_date: null, overall_effectiveness: null, short_inspection_date: null} + + - name: stg_ofsted_fe_inspections_parses_dates_and_safeguarding + model: stg_ofsted_fe_inspections + given: + - input: source('raw', 'ofsted_fe_inspections') + rows: + - {inspection_number: '10380001', provider_urn: '143526', provider_ukprn: '10000002', first_day_of_inspection: '12/05/2026', date_published: '16/06/2026', safeguarding_standards: 'Met'} + - {inspection_number: '10380002', provider_urn: '150042', provider_ukprn: '10000003', first_day_of_inspection: '20/01/2026', date_published: '24/02/2026', safeguarding_standards: 'Not met'} + - {inspection_number: '10380003', provider_urn: '1', provider_ukprn: '', first_day_of_inspection: '', date_published: '', safeguarding_standards: ''} + expect: + rows: + - {inspection_number: '10380001', provider_urn: 143526, rc_inspection_date: '2026-05-12', published_date: '2026-06-16', rc_safeguarding_met: true} + - {inspection_number: '10380002', provider_urn: 150042, rc_inspection_date: '2026-01-20', published_date: '2026-02-24', rc_safeguarding_met: false} diff --git a/pipeline/transform/models/staging/stg_ofsted_fe_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_fe_inspections.sql new file mode 100644 index 0000000..c4a04ee --- /dev/null +++ b/pipeline/transform/models/staging/stg_ofsted_fe_inspections.sql @@ -0,0 +1,18 @@ +-- Staging model: Ofsted's renewed-framework FE inspections (report cards), +-- one row per inspection (audit H4). Only the headline is kept: the date and +-- the safeguarding result. + +select + trim(inspection_number) as inspection_number, + cast(trim(provider_urn) as integer) as provider_urn, + nullif(nullif(trim(provider_ukprn), ''), '-') as provider_ukprn, + to_date(trim(first_day_of_inspection), 'DD/MM/YYYY') as rc_inspection_date, + case when trim(date_published) ~ '^\d{2}/\d{2}/\d{4}$' + then to_date(trim(date_published), 'DD/MM/YYYY') end as published_date, + case lower(trim(safeguarding_standards)) + when 'met' then true + when 'not met' then false + end as rc_safeguarding_met +from {{ source('raw', 'ofsted_fe_inspections') }} +where trim(provider_urn) ~ '^[0-9]+$' + and trim(first_day_of_inspection) ~ '^\d{2}/\d{2}/\d{4}$' diff --git a/pipeline/transform/models/staging/stg_ofsted_fe_providers.sql b/pipeline/transform/models/staging/stg_ofsted_fe_providers.sql new file mode 100644 index 0000000..3d205df --- /dev/null +++ b/pipeline/transform/models/staging/stg_ofsted_fe_providers.sql @@ -0,0 +1,18 @@ +-- Staging model: Ofsted further education and skills providers — the "as at" +-- MI file, one row per provider with its latest inspection under the previous +-- framework (audit H4). Ofsted writes '-' for no grade or date. + +select + cast(trim(provider_urn) as integer) as provider_urn, + nullif(nullif(trim(provider_ukprn), ''), '-') as provider_ukprn, + provider_name, + provider_type, + lower(trim(renewed_eif_full_inspection)) = 'yes' as renewed_eif_full_inspection, + case when trim(graded_inspection_date) ~ '^\d{2}/\d{2}/\d{4}$' + then to_date(trim(graded_inspection_date), 'DD/MM/YYYY') end as graded_inspection_date, + case when trim(overall_effectiveness) in ('1', '2', '3', '4') + then trim(overall_effectiveness)::integer end as overall_effectiveness, + case when trim(short_inspection_date) ~ '^\d{2}/\d{2}/\d{4}$' + then to_date(trim(short_inspection_date), 'DD/MM/YYYY') end as short_inspection_date +from {{ source('raw', 'ofsted_fe_providers') }} +where trim(provider_urn) ~ '^[0-9]+$'