diff --git a/backend/data_loader.py b/backend/data_loader.py index 2fbaab2..ce51ab1 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -204,7 +204,8 @@ _MAIN_QUERY = text(""" LEFT JOIN ( SELECT DISTINCT ON (urn) urn, - overall_effectiveness AS ofsted_grade, + -- Fall back to the ungraded-inspection grade when no graded grade exists. + COALESCE(overall_effectiveness, ungraded_grade) AS ofsted_grade, inspection_date AS ofsted_date, framework AS ofsted_framework FROM marts.fact_ofsted_inspection diff --git a/backend/models.py b/backend/models.py index 94b9d2e..18f7936 100644 --- a/backend/models.py +++ b/backend/models.py @@ -133,6 +133,10 @@ class FactOfstedInspection(Base): leadership_management = Column(Integer) early_years_provision = Column(Integer) sixth_form_provision = Column(Integer) + # Ungraded (Section 8) inspection: raw outcome text and the grade parsed from + # it (fallback for schools with no graded overall effectiveness). + ungraded_outcome = Column(String(100)) + ungraded_grade = Column(Integer) rc_safeguarding_met = Column(Boolean) rc_inclusion = Column(Integer) rc_curriculum_teaching = Column(Integer) diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index 764bff5..168d88d 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -59,6 +59,15 @@ COLUMN_PRIORITY = { "Latest OEIF sixth form provision", "Sixth form provision (where applicable)", ], + # Ungraded (Section 8) inspection — no fresh grade is assigned; the outcome + # is free text like "School remains Good". Used as a last-resort fallback + # for the school's grade when no graded overall effectiveness is available. + "ungraded_outcome": [ + "Ungraded inspection overall outcome", + ], + "ungraded_inspection_date": [ + "Date of latest ungraded inspection", + ], } @@ -100,6 +109,8 @@ class OfstedInspectionsStream(Stream): th.Property("effectiveness_of_leadership_and_management", th.StringType), th.Property("early_years_provision", th.StringType), th.Property("sixth_form_provision", th.StringType), + th.Property("ungraded_outcome", th.StringType), + th.Property("ungraded_inspection_date", th.StringType), th.Property("report_url", th.StringType), ).to_dict() diff --git a/pipeline/transform/macros/parse_ungraded_outcome.sql b/pipeline/transform/macros/parse_ungraded_outcome.sql new file mode 100644 index 0000000..47e243a --- /dev/null +++ b/pipeline/transform/macros/parse_ungraded_outcome.sql @@ -0,0 +1,20 @@ +-- Macro: Parse an Ofsted ungraded (Section 8) inspection outcome into a grade. +-- +-- Ungraded inspections do not assign a fresh grade; the outcome is free text +-- describing whether the school kept its previous judgement, e.g. +-- "School remains Good", "School remains Good (Concerns) - S5 Next", +-- "School remains Outstanding", "School remains Outstanding (Concerns) - S5 Next". +-- These are the only Good/Outstanding signal we can recover, so we match on the +-- "remains " phrase (tolerant of any trailing qualifier). +-- +-- Outcomes that don't establish a Good/Outstanding status — "Standards +-- maintained", "Improved significantly", "Some aspects not as strong" — map to +-- null. Grades align with parse_ofsted_grade (1 = Outstanding, 2 = Good). + +{% macro parse_ungraded_outcome(column) %} + case + when {{ column }} ilike '%remains Outstanding%' then 1 + when {{ column }} ilike '%remains Good%' then 2 + else null + end +{% endmacro %} diff --git a/pipeline/transform/models/intermediate/int_ofsted_latest.sql b/pipeline/transform/models/intermediate/int_ofsted_latest.sql index d0cb10a..0e5d88e 100644 --- a/pipeline/transform/models/intermediate/int_ofsted_latest.sql +++ b/pipeline/transform/models/intermediate/int_ofsted_latest.sql @@ -23,6 +23,8 @@ select leadership_management, early_years_provision, sixth_form_provision, + ungraded_outcome, + ungraded_grade, rc_safeguarding_met, rc_inclusion, rc_curriculum_teaching, diff --git a/pipeline/transform/models/marts/dim_school.sql b/pipeline/transform/models/marts/dim_school.sql index b423cea..3b47d47 100644 --- a/pipeline/transform/models/marts/dim_school.sql +++ b/pipeline/transform/models/marts/dim_school.sql @@ -60,7 +60,9 @@ select -- Latest Ofsted (populated after monthly Ofsted pipeline runs) {% if ofsted_relation is not none %} - o.overall_effectiveness as ofsted_grade, + -- Prefer the graded overall effectiveness; fall back to the grade parsed + -- from the latest ungraded (Section 8) outcome when no graded grade exists. + coalesce(o.overall_effectiveness, o.ungraded_grade) as ofsted_grade, o.inspection_date as ofsted_date, o.framework as ofsted_framework {% else %} diff --git a/pipeline/transform/models/marts/fact_ofsted_inspection.sql b/pipeline/transform/models/marts/fact_ofsted_inspection.sql index 57d1a94..f96a126 100644 --- a/pipeline/transform/models/marts/fact_ofsted_inspection.sql +++ b/pipeline/transform/models/marts/fact_ofsted_inspection.sql @@ -12,6 +12,8 @@ select leadership_management, early_years_provision, sixth_form_provision, + ungraded_outcome, + ungraded_grade, rc_safeguarding_met, rc_inclusion, rc_curriculum_teaching, diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_inspections.sql index 100e6e1..557d1bb 100644 --- a/pipeline/transform/models/staging/stg_ofsted_inspections.sql +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.sql @@ -8,7 +8,13 @@ with source as ( renamed as ( select cast(urn as integer) as urn, - to_date(nullif(trim(inspection_date), 'NULL'), 'DD/MM/YYYY') as inspection_date, + -- Inspection event date: the graded inspection when present, otherwise the + -- ungraded (Section 8) inspection so schools with only an ungraded + -- inspection are still retained. + coalesce( + to_date(nullif(trim(inspection_date), 'NULL'), 'DD/MM/YYYY'), + to_date(nullif(trim(ungraded_inspection_date), 'NULL'), 'DD/MM/YYYY') + ) as inspection_date, inspection_type, event_type_grouping as framework, @@ -21,6 +27,12 @@ renamed as ( {{ safe_numeric('early_years_provision') }}::integer as early_years_provision, {{ safe_numeric('sixth_form_provision') }}::integer as sixth_form_provision, + -- Ungraded (Section 8) inspection outcome — free text, plus a grade + -- parsed from it (1/2/null) used as a last-resort fallback for schools + -- with no graded overall effectiveness. + nullif(trim(ungraded_outcome), 'NULL') as ungraded_outcome, + {{ parse_ungraded_outcome('ungraded_outcome') }}::integer as ungraded_grade, + -- Report Card fields (post-Nov 2025 framework) -- TODO: add rc_* columns to tap-uk-ofsted schema once CSV column names are confirmed null::text as rc_safeguarding_met, @@ -36,7 +48,10 @@ renamed as ( report_url from source where urn is not null - and nullif(trim(inspection_date), 'NULL') is not null + and ( + nullif(trim(inspection_date), 'NULL') is not null + or nullif(trim(ungraded_inspection_date), 'NULL') is not null + ) ) select * from renamed