From 8e82e2e4cfbd5d7dd5cefce1c1a8e9e73874989d Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 13 Oct 2026 21:26:42 +0100 Subject: [PATCH] feat(dbt): dim_school.inspectorate and ofsted_listed (H4) Co-Authored-By: Claude Opus 5.5 --- pipeline/dags/school_data_pipeline.py | 2 +- pipeline/tests/test_dag_selectors.py | 6 ++-- .../models/intermediate/int_ofsted_listed.sql | 21 ++++++++++++++ .../models/intermediate/int_ofsted_listed.yml | 29 +++++++++++++++++++ .../transform/models/marts/dim_school.sql | 24 +++++++++++++-- .../transform/models/marts/dim_school.yml | 26 +++++++++++++++++ 6 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 pipeline/transform/models/intermediate/int_ofsted_listed.sql create mode 100644 pipeline/transform/models/intermediate/int_ofsted_listed.yml diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 39dc868..da4a4b6 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -146,7 +146,7 @@ with DAG( dbt_build_ofsted = BashOperator( task_id="dbt_build", - bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+", + bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ stg_ofsted_fe_providers+ stg_ofsted_fe_inspections+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+", ) sync_typesense_ofsted = BashOperator( diff --git a/pipeline/tests/test_dag_selectors.py b/pipeline/tests/test_dag_selectors.py index c41a55e..0d15846 100644 --- a/pipeline/tests/test_dag_selectors.py +++ b/pipeline/tests/test_dag_selectors.py @@ -24,9 +24,9 @@ DAG_ID = re.compile(r'dag_id="([a-z0-9_]+)"') DAILY = 'school_data_daily' -# dim_school reads int_ofsted_latest only when the relation exists -# (adapter.get_relation), so a missing table is not a failure. -OPTIONAL_PARENTS = {'int_ofsted_latest'} +# dim_school reads int_ofsted_latest and int_ofsted_listed only when the +# relations exist (adapter.get_relation), so a missing table is not a failure. +OPTIONAL_PARENTS = {'int_ofsted_latest', 'int_ofsted_listed'} def model_parents(): diff --git a/pipeline/transform/models/intermediate/int_ofsted_listed.sql b/pipeline/transform/models/intermediate/int_ofsted_listed.sql new file mode 100644 index 0000000..e4702ae --- /dev/null +++ b/pipeline/transform/models/intermediate/int_ofsted_listed.sql @@ -0,0 +1,21 @@ +-- Intermediate model: every URN Ofsted's own data lists, inspected or not +-- (audit H4). Only these may be called "Not yet inspected". Reads the raw +-- school MI because stg_ofsted_inspections drops rows with no inspection date. + +with school_mi as ( + select distinct cast(urn as integer) as urn + from {{ source('raw', 'ofsted_inspections') }} + where urn is not null +), + +fe as ( + select distinct g.urn + from {{ ref('stg_gias_ukprn') }} g + join {{ ref('stg_ofsted_fe_providers') }} p + on p.provider_urn = g.urn + or (g.ukprn is not null and p.provider_ukprn = g.ukprn) +) + +select urn from school_mi +union +select urn from fe diff --git a/pipeline/transform/models/intermediate/int_ofsted_listed.yml b/pipeline/transform/models/intermediate/int_ofsted_listed.yml new file mode 100644 index 0000000..b90279d --- /dev/null +++ b/pipeline/transform/models/intermediate/int_ofsted_listed.yml @@ -0,0 +1,29 @@ +version: 2 + +unit_tests: + - name: int_ofsted_listed_includes_uninspected_schools_and_fe_providers + description: > + Abbas and Templecombe (150031) is in Ofsted's school MI with no inspection; + Exeter and North Devon Colleges Group (130645) is in the FE MI with none; + a UKPRN-only FE match counts too. A school in neither is absent (audit H4). + model: int_ofsted_listed + given: + - input: source('raw', 'ofsted_inspections') + rows: + - {urn: '150031', inspection_date: 'NULL'} + - {urn: '102408', inspection_date: '17/06/2025'} + - input: ref('stg_gias_ukprn') + rows: + - {urn: 130645, ukprn: null} + - {urn: 5, ukprn: '10099999'} + - {urn: 133058, ukprn: null} + - input: ref('stg_ofsted_fe_providers') + rows: + - {provider_urn: 130645, provider_ukprn: null} + - {provider_urn: 9999999, provider_ukprn: '10099999'} + expect: + rows: + - {urn: 150031} + - {urn: 102408} + - {urn: 130645} + - {urn: 5} diff --git a/pipeline/transform/models/marts/dim_school.sql b/pipeline/transform/models/marts/dim_school.sql index f11f5c8..11d3a0c 100644 --- a/pipeline/transform/models/marts/dim_school.sql +++ b/pipeline/transform/models/marts/dim_school.sql @@ -1,9 +1,10 @@ -- Mart: Canonical school dimension — one row per active URN --- int_ofsted_latest is only ref()'d inside a conditional block below (it may not --- exist before the Ofsted pipeline first runs), so declare the dependency --- explicitly for dbt's DAG. +-- int_ofsted_latest and int_ofsted_listed are only ref()'d inside conditional +-- blocks below (they may not exist before the Ofsted pipeline first runs), so +-- declare the dependencies explicitly for dbt's DAG. -- depends_on: {{ ref('int_ofsted_latest') }} +-- depends_on: {{ ref('int_ofsted_listed') }} with schools as ( select * from {{ ref('stg_gias_establishments') }} @@ -15,6 +16,12 @@ with schools as ( identifier='int_ofsted_latest' ) %} +{% set listed_relation = adapter.get_relation( + database=target.database, + schema='intermediate', + identifier='int_ofsted_listed' +) %} + select s.urn, s.local_authority_code * 1000 + s.establishment_number as laestab, @@ -70,6 +77,14 @@ select s.status_code, s.nursery_provision, s.admissions_policy_code, + -- The inspectorate GIAS names ("ISI" for most independent schools) and + -- whether Ofsted's own data lists the school, inspected or not (audit H4). + s.inspectorate_name as inspectorate, + {% if listed_relation is not none %} + l.urn is not null as ofsted_listed, + {% else %} + null::boolean as ofsted_listed, + {% endif %} -- Latest Ofsted (populated after monthly Ofsted pipeline runs). The grade -- still in force and the latest visit, from int_ofsted_latest — never a @@ -88,6 +103,9 @@ from schools s {% if ofsted_relation is not none %} left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn {% endif %} +{% if listed_relation is not none %} +left join {{ ref('int_ofsted_listed') }} l on s.urn = l.urn +{% endif %} -- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when -- GIAS flips to Closed — marts fully rebuild each run). where s.status_code in (1, 3) diff --git a/pipeline/transform/models/marts/dim_school.yml b/pipeline/transform/models/marts/dim_school.yml index 8b5d555..0d51286 100644 --- a/pipeline/transform/models/marts/dim_school.yml +++ b/pipeline/transform/models/marts/dim_school.yml @@ -14,6 +14,8 @@ unit_tests: - input: ref('int_ofsted_latest') rows: - {urn: 102408, inspection_date: '2025-06-17', overall_effectiveness: null, ungraded_grade: 2, framework: 'Schools - S5', latest_visit_date: '2025-06-17', current_grade: null} + - input: ref('int_ofsted_listed') + rows: [] expect: rows: - {urn: 102408, ofsted_grade: null, ofsted_date: '2025-06-17', ofsted_framework: 'Schools - S5'} @@ -30,6 +32,30 @@ unit_tests: - input: ref('int_ofsted_latest') rows: - {urn: 139888, inspection_date: '2020-03-03', overall_effectiveness: 2, ungraded_grade: null, framework: 'Schools - S5', latest_visit_date: '2025-05-21', current_grade: 2} + - input: ref('int_ofsted_listed') + rows: [] expect: rows: - {urn: 139888, ofsted_grade: 2, ofsted_date: '2025-05-21'} + + - name: dim_school_carries_inspectorate_and_ofsted_listing + description: > + Audit H4. GIAS names ISI for Abbey College Cambridge; Ofsted lists Abbas + and Templecombe without an inspection; a third school is in neither. + model: dim_school + given: + - input: ref('stg_gias_establishments') + rows: + - {urn: 136083, school_name: 'Abbey College Cambridge', status_code: 1, school_type_code: 11, local_authority_code: 873, phase_code: 4, inspectorate_name: 'ISI'} + - {urn: 150031, school_name: 'Abbas and Templecombe Church School', status_code: 1, school_type_code: 28, local_authority_code: 933, phase_code: 2, inspectorate_name: 'Not applicable'} + - {urn: 1, school_name: 'No data school', status_code: 1, school_type_code: 1, local_authority_code: 312, phase_code: 2, inspectorate_name: null} + - input: ref('int_ofsted_latest') + rows: [] + - input: ref('int_ofsted_listed') + rows: + - {urn: 150031} + expect: + rows: + - {urn: 136083, inspectorate: 'ISI', ofsted_listed: false} + - {urn: 150031, inspectorate: 'Not applicable', ofsted_listed: true} + - {urn: 1, inspectorate: null, ofsted_listed: false}