feat(dbt): dim_school.inspectorate and ofsted_listed (H4)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

pipeline/dags/school_data_pipeline.py

@@ -146,7 +146,7 @@ with DAG(
 
     dbt_build_ofsted = BashOperator(
         task_id="dbt_build",
-        bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
+        bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ stg_ofsted_fe_providers+ stg_ofsted_fe_inspections+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
     )
 
     sync_typesense_ofsted = BashOperator(

pipeline/tests/test_dag_selectors.py

@@ -24,9 +24,9 @@ DAG_ID = re.compile(r'dag_id="([a-z0-9_]+)"')
 
 DAILY = 'school_data_daily'
 
-# dim_school reads int_ofsted_latest only when the relation exists
-# (adapter.get_relation), so a missing table is not a failure.
-OPTIONAL_PARENTS = {'int_ofsted_latest'}
+# dim_school reads int_ofsted_latest and int_ofsted_listed only when the
+# relations exist (adapter.get_relation), so a missing table is not a failure.
+OPTIONAL_PARENTS = {'int_ofsted_latest', 'int_ofsted_listed'}
 
 
 def model_parents():

pipeline/transform/models/intermediate/int_ofsted_listed.sql

@@ -0,0 +1,21 @@
+-- Intermediate model: every URN Ofsted's own data lists, inspected or not
+-- (audit H4). Only these may be called "Not yet inspected". Reads the raw
+-- school MI because stg_ofsted_inspections drops rows with no inspection date.
+
+with school_mi as (
+    select distinct cast(urn as integer) as urn
+    from {{ source('raw', 'ofsted_inspections') }}
+    where urn is not null
+),
+
+fe as (
+    select distinct g.urn
+    from {{ ref('stg_gias_ukprn') }} g
+    join {{ ref('stg_ofsted_fe_providers') }} p
+      on p.provider_urn = g.urn
+      or (g.ukprn is not null and p.provider_ukprn = g.ukprn)
+)
+
+select urn from school_mi
+union
+select urn from fe

pipeline/transform/models/intermediate/int_ofsted_listed.yml

@@ -0,0 +1,29 @@
+version: 2
+
+unit_tests:
+  - name: int_ofsted_listed_includes_uninspected_schools_and_fe_providers
+    description: >
+      Abbas and Templecombe (150031) is in Ofsted's school MI with no inspection;
+      Exeter and North Devon Colleges Group (130645) is in the FE MI with none;
+      a UKPRN-only FE match counts too. A school in neither is absent (audit H4).
+    model: int_ofsted_listed
+    given:
+      - input: source('raw', 'ofsted_inspections')
+        rows:
+          - {urn: '150031', inspection_date: 'NULL'}
+          - {urn: '102408', inspection_date: '17/06/2025'}
+      - input: ref('stg_gias_ukprn')
+        rows:
+          - {urn: 130645, ukprn: null}
+          - {urn: 5, ukprn: '10099999'}
+          - {urn: 133058, ukprn: null}
+      - input: ref('stg_ofsted_fe_providers')
+        rows:
+          - {provider_urn: 130645, provider_ukprn: null}
+          - {provider_urn: 9999999, provider_ukprn: '10099999'}
+    expect:
+      rows:
+        - {urn: 150031}
+        - {urn: 102408}
+        - {urn: 130645}
+        - {urn: 5}

pipeline/transform/models/marts/dim_school.sql

@@ -1,9 +1,10 @@
 -- Mart: Canonical school dimension — one row per active URN
 
--- int_ofsted_latest is only ref()'d inside a conditional block below (it may not
--- exist before the Ofsted pipeline first runs), so declare the dependency
--- explicitly for dbt's DAG.
+-- int_ofsted_latest and int_ofsted_listed are only ref()'d inside conditional
+-- blocks below (they may not exist before the Ofsted pipeline first runs), so
+-- declare the dependencies explicitly for dbt's DAG.
 -- depends_on: {{ ref('int_ofsted_latest') }}
+-- depends_on: {{ ref('int_ofsted_listed') }}
 
 with schools as (
     select * from {{ ref('stg_gias_establishments') }}
@@ -15,6 +16,12 @@ with schools as (
     identifier='int_ofsted_latest'
 ) %}
 
+{% set listed_relation = adapter.get_relation(
+    database=target.database,
+    schema='intermediate',
+    identifier='int_ofsted_listed'
+) %}
+
 select
     s.urn,
     s.local_authority_code * 1000 + s.establishment_number as laestab,
@@ -70,6 +77,14 @@ select
     s.status_code,
     s.nursery_provision,
     s.admissions_policy_code,
+    -- The inspectorate GIAS names ("ISI" for most independent schools) and
+    -- whether Ofsted's own data lists the school, inspected or not (audit H4).
+    s.inspectorate_name as inspectorate,
+    {% if listed_relation is not none %}
+    l.urn is not null as ofsted_listed,
+    {% else %}
+    null::boolean as ofsted_listed,
+    {% endif %}
 
     -- Latest Ofsted (populated after monthly Ofsted pipeline runs). The grade
     -- still in force and the latest visit, from int_ofsted_latest — never a
@@ -88,6 +103,9 @@ from schools s
 {% if ofsted_relation is not none %}
 left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn
 {% endif %}
+{% if listed_relation is not none %}
+left join {{ ref('int_ofsted_listed') }} l on s.urn = l.urn
+{% endif %}
 -- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when
 -- GIAS flips to Closed — marts fully rebuild each run).
 where s.status_code in (1, 3)

pipeline/transform/models/marts/dim_school.yml

@@ -14,6 +14,8 @@ unit_tests:
       - input: ref('int_ofsted_latest')
         rows:
           - {urn: 102408, inspection_date: '2025-06-17', overall_effectiveness: null, ungraded_grade: 2, framework: 'Schools - S5', latest_visit_date: '2025-06-17', current_grade: null}
+      - input: ref('int_ofsted_listed')
+        rows: []
     expect:
       rows:
         - {urn: 102408, ofsted_grade: null, ofsted_date: '2025-06-17', ofsted_framework: 'Schools - S5'}
@@ -30,6 +32,30 @@ unit_tests:
       - input: ref('int_ofsted_latest')
         rows:
           - {urn: 139888, inspection_date: '2020-03-03', overall_effectiveness: 2, ungraded_grade: null, framework: 'Schools - S5', latest_visit_date: '2025-05-21', current_grade: 2}
+      - input: ref('int_ofsted_listed')
+        rows: []
     expect:
       rows:
         - {urn: 139888, ofsted_grade: 2, ofsted_date: '2025-05-21'}
+
+  - name: dim_school_carries_inspectorate_and_ofsted_listing
+    description: >
+      Audit H4. GIAS names ISI for Abbey College Cambridge; Ofsted lists Abbas
+      and Templecombe without an inspection; a third school is in neither.
+    model: dim_school
+    given:
+      - input: ref('stg_gias_establishments')
+        rows:
+          - {urn: 136083, school_name: 'Abbey College Cambridge', status_code: 1, school_type_code: 11, local_authority_code: 873, phase_code: 4, inspectorate_name: 'ISI'}
+          - {urn: 150031, school_name: 'Abbas and Templecombe Church School', status_code: 1, school_type_code: 28, local_authority_code: 933, phase_code: 2, inspectorate_name: 'Not applicable'}
+          - {urn: 1, school_name: 'No data school', status_code: 1, school_type_code: 1, local_authority_code: 312, phase_code: 2, inspectorate_name: null}
+      - input: ref('int_ofsted_latest')
+        rows: []
+      - input: ref('int_ofsted_listed')
+        rows:
+          - {urn: 150031}
+    expect:
+      rows:
+        - {urn: 136083, inspectorate: 'ISI', ofsted_listed: false}
+        - {urn: 150031, inspectorate: 'Not applicable', ofsted_listed: true}
+        - {urn: 1, inspectorate: null, ofsted_listed: false}