feat(ofsted): fall back to ungraded inspection outcome for school grade

Ungraded (Section 8) inspections don't assign a fresh grade — the export
only gives free text like "School remains Good". Parse that text into a
grade (remains Outstanding -> 1, remains Good -> 2, else null) and use it
as a last-resort fallback when no graded overall effectiveness exists.

Also retain schools that have only an ungraded inspection (no graded date)
by coalescing the inspection date, so ~8.5k previously-dropped schools now
carry a grade.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/data_loader.py

@@ -204,7 +204,8 @@ _MAIN_QUERY = text("""
     LEFT JOIN (
         SELECT DISTINCT ON (urn)
             urn,
-            overall_effectiveness AS ofsted_grade,
+            -- Fall back to the ungraded-inspection grade when no graded grade exists.
+            COALESCE(overall_effectiveness, ungraded_grade) AS ofsted_grade,
             inspection_date       AS ofsted_date,
             framework             AS ofsted_framework
         FROM marts.fact_ofsted_inspection

backend/models.py

@@ -133,6 +133,10 @@ class FactOfstedInspection(Base):
     leadership_management = Column(Integer)
     early_years_provision = Column(Integer)
     sixth_form_provision = Column(Integer)
+    # Ungraded (Section 8) inspection: raw outcome text and the grade parsed from
+    # it (fallback for schools with no graded overall effectiveness).
+    ungraded_outcome = Column(String(100))
+    ungraded_grade = Column(Integer)
     rc_safeguarding_met = Column(Boolean)
     rc_inclusion = Column(Integer)
     rc_curriculum_teaching = Column(Integer)

pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py

@@ -59,6 +59,15 @@ COLUMN_PRIORITY = {
         "Latest OEIF sixth form provision",
         "Sixth form provision (where applicable)",
     ],
+    # Ungraded (Section 8) inspection — no fresh grade is assigned; the outcome
+    # is free text like "School remains Good". Used as a last-resort fallback
+    # for the school's grade when no graded overall effectiveness is available.
+    "ungraded_outcome": [
+        "Ungraded inspection overall outcome",
+    ],
+    "ungraded_inspection_date": [
+        "Date of latest ungraded inspection",
+    ],
 }
 
 
@@ -100,6 +109,8 @@ class OfstedInspectionsStream(Stream):
         th.Property("effectiveness_of_leadership_and_management", th.StringType),
         th.Property("early_years_provision", th.StringType),
         th.Property("sixth_form_provision", th.StringType),
+        th.Property("ungraded_outcome", th.StringType),
+        th.Property("ungraded_inspection_date", th.StringType),
         th.Property("report_url", th.StringType),
     ).to_dict()
 

pipeline/transform/macros/parse_ungraded_outcome.sql

@@ -0,0 +1,20 @@
+-- Macro: Parse an Ofsted ungraded (Section 8) inspection outcome into a grade.
+--
+-- Ungraded inspections do not assign a fresh grade; the outcome is free text
+-- describing whether the school kept its previous judgement, e.g.
+--   "School remains Good", "School remains Good (Concerns) - S5 Next",
+--   "School remains Outstanding", "School remains Outstanding (Concerns) - S5 Next".
+-- These are the only Good/Outstanding signal we can recover, so we match on the
+-- "remains <grade>" phrase (tolerant of any trailing qualifier).
+--
+-- Outcomes that don't establish a Good/Outstanding status — "Standards
+-- maintained", "Improved significantly", "Some aspects not as strong" — map to
+-- null. Grades align with parse_ofsted_grade (1 = Outstanding, 2 = Good).
+
+{% macro parse_ungraded_outcome(column) %}
+    case
+        when {{ column }} ilike '%remains Outstanding%' then 1
+        when {{ column }} ilike '%remains Good%'        then 2
+        else null
+    end
+{% endmacro %}

pipeline/transform/models/intermediate/int_ofsted_latest.sql

@@ -23,6 +23,8 @@ select
     leadership_management,
     early_years_provision,
     sixth_form_provision,
+    ungraded_outcome,
+    ungraded_grade,
     rc_safeguarding_met,
     rc_inclusion,
     rc_curriculum_teaching,

pipeline/transform/models/marts/dim_school.sql

@@ -60,7 +60,9 @@ select
 
     -- Latest Ofsted (populated after monthly Ofsted pipeline runs)
     {% if ofsted_relation is not none %}
-    o.overall_effectiveness as ofsted_grade,
+    -- Prefer the graded overall effectiveness; fall back to the grade parsed
+    -- from the latest ungraded (Section 8) outcome when no graded grade exists.
+    coalesce(o.overall_effectiveness, o.ungraded_grade) as ofsted_grade,
     o.inspection_date as ofsted_date,
     o.framework as ofsted_framework
     {% else %}

pipeline/transform/models/marts/fact_ofsted_inspection.sql

@@ -12,6 +12,8 @@ select
     leadership_management,
     early_years_provision,
     sixth_form_provision,
+    ungraded_outcome,
+    ungraded_grade,
     rc_safeguarding_met,
     rc_inclusion,
     rc_curriculum_teaching,

pipeline/transform/models/staging/stg_ofsted_inspections.sql

@@ -8,7 +8,13 @@ with source as (
 renamed as (
     select
         cast(urn as integer)                                                    as urn,
-        to_date(nullif(trim(inspection_date), 'NULL'), 'DD/MM/YYYY')             as inspection_date,
+        -- Inspection event date: the graded inspection when present, otherwise the
+        -- ungraded (Section 8) inspection so schools with only an ungraded
+        -- inspection are still retained.
+        coalesce(
+            to_date(nullif(trim(inspection_date), 'NULL'), 'DD/MM/YYYY'),
+            to_date(nullif(trim(ungraded_inspection_date), 'NULL'), 'DD/MM/YYYY')
+        )                                                                       as inspection_date,
         inspection_type,
         event_type_grouping                                                     as framework,
 
@@ -21,6 +27,12 @@ renamed as (
         {{ safe_numeric('early_years_provision') }}::integer                    as early_years_provision,
         {{ safe_numeric('sixth_form_provision') }}::integer                     as sixth_form_provision,
 
+        -- Ungraded (Section 8) inspection outcome — free text, plus a grade
+        -- parsed from it (1/2/null) used as a last-resort fallback for schools
+        -- with no graded overall effectiveness.
+        nullif(trim(ungraded_outcome), 'NULL')                                  as ungraded_outcome,
+        {{ parse_ungraded_outcome('ungraded_outcome') }}::integer               as ungraded_grade,
+
         -- Report Card fields (post-Nov 2025 framework)
         -- TODO: add rc_* columns to tap-uk-ofsted schema once CSV column names are confirmed
         null::text as rc_safeguarding_met,
@@ -36,7 +48,10 @@ renamed as (
         report_url
     from source
     where urn is not null
-      and nullif(trim(inspection_date), 'NULL') is not null
+      and (
+          nullif(trim(inspection_date), 'NULL') is not null
+          or nullif(trim(ungraded_inspection_date), 'NULL') is not null
+      )
 )
 
 select * from renamed