From 05cb22f1a500bd2fa5bfc5335681990c0bfac38b Mon Sep 17 00:00:00 2001 From: Tudor Date: Fri, 27 Mar 2026 17:23:46 +0000 Subject: [PATCH] fix(stg_ofsted_inspections): handle NULL strings from Ofsted CSV Use nullif+trim for date cast and safe_numeric for integer grades to handle literal 'NULL' strings present in the new Report Card format CSV. Co-Authored-By: Claude Sonnet 4.6 --- .../models/staging/stg_ofsted_inspections.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_inspections.sql index 0a88467..54eae5c 100644 --- a/pipeline/transform/models/staging/stg_ofsted_inspections.sql +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.sql @@ -7,19 +7,19 @@ with source as ( renamed as ( select - cast(urn as integer) as urn, - cast(inspection_date as date) as inspection_date, + cast(urn as integer) as urn, + nullif(trim(inspection_date), 'NULL')::date as inspection_date, inspection_type, - event_type_grouping as framework, + event_type_grouping as framework, -- OEIF grades (1-4 scale) - cast(overall_effectiveness as integer) as overall_effectiveness, - cast(quality_of_education as integer) as quality_of_education, - cast(behaviour_and_attitudes as integer) as behaviour_attitudes, - cast(personal_development as integer) as personal_development, - cast(effectiveness_of_leadership_and_management as integer) as leadership_management, - cast(early_years_provision as integer) as early_years_provision, - cast(sixth_form_provision as integer) as sixth_form_provision, + {{ safe_numeric('overall_effectiveness') }}::integer as overall_effectiveness, + {{ safe_numeric('quality_of_education') }}::integer as quality_of_education, + {{ safe_numeric('behaviour_and_attitudes') }}::integer as behaviour_attitudes, + {{ safe_numeric('personal_development') }}::integer as personal_development, + {{ safe_numeric('effectiveness_of_leadership_and_management') }}::integer as leadership_management, + {{ safe_numeric('early_years_provision') }}::integer as early_years_provision, + {{ safe_numeric('sixth_form_provision') }}::integer as sixth_form_provision, -- Report Card fields (post-Nov 2025 framework) -- TODO: add rc_* columns to tap-uk-ofsted schema once CSV column names are confirmed