diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index f649d57..2909136 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -1545,6 +1545,32 @@ test('searching a Welsh authority by name returns no schools', async ({ page }) expect(body.total ?? (body.schools ?? []).length).toBe(0); }); +test('a Welsh school GIAS does not type as Welsh is gone too', async ({ page }) => { + // Beechwood College, Sully, CF64 5SE. GIAS types it "Special post 16 + // institution" (32), not "Welsh establishment" (30), so the establishment + // type filter alone left it behind — it was the last Welsh school standing. + const res = await page.request.get('/api/schools?local_authority=Vale%20of%20Glamorgan&page_size=1'); + expect(res.ok()).toBeTruthy(); + const body = await res.json(); + expect(body.total ?? (body.schools ?? []).length).toBe(0); + + const page404 = await page.goto('/school/142458-beechwood-college'); + expect(page404?.status()).toBe(404); +}); + +test('English schools with Welsh postcodes are kept', async ({ page }) => { + // Royal Mail postcode areas straddle the border: these Gloucestershire + // schools carry NP16/NP25 postcodes but are English schools with English + // data. A postcode-based filter would have wrongly deleted them, so this + // guards the fix from being "simplified" into one. + for (const urn of [115639, 115655, 137382]) { + const res = await page.request.get(`/api/schools/${urn}`); + expect(res.ok(), `URN ${urn} should still resolve`).toBeTruthy(); + const { school_info } = await res.json(); + expect(school_info.local_authority).toBe('Gloucestershire'); + } +}); + test('a Welsh school URL 404s while an English one still resolves', async ({ page }) => { // Paired on purpose: the Welsh assertion alone would also pass if the whole // site were down, which is the failure this test most needs to distinguish. diff --git a/pipeline/transform/dbt_project.yml b/pipeline/transform/dbt_project.yml index ff62b6f..bd2a363 100644 --- a/pipeline/transform/dbt_project.yml +++ b/pipeline/transform/dbt_project.yml @@ -24,6 +24,19 @@ clean-targets: ["target", "dbt_packages"] vars: non_england_school_type_codes: [25, 26, 30, 37] + # Type alone is not enough. Beechwood College (URN 142458, Sully, CF64 5SE) + # is a Welsh school that GIAS types as a Special post 16 institution (32) + # rather than a Welsh establishment (30), so the type filter let it through. + # LA code is the reliable discriminator: GIAS gives the 22 Welsh unitary + # authorities the contiguous block 660-681, which English authorities never + # use (they occupy 201-384 and 800-938). + # + # Filtering on postcode would be wrong: Redbrook, Tutshill, Wyedean and two + # other Gloucestershire schools carry NP16/NP25 postcodes because Royal Mail + # areas straddle the border, and they are English schools with English data. + welsh_la_code_min: 660 + welsh_la_code_max: 681 + models: school_compare: staging: diff --git a/pipeline/transform/models/marts/dim_location.sql b/pipeline/transform/models/marts/dim_location.sql index 3495b14..0b7ad56 100644 --- a/pipeline/transform/models/marts/dim_location.sql +++ b/pipeline/transform/models/marts/dim_location.sql @@ -37,3 +37,5 @@ where s.status_code in (1, 3) -- coalesce, not a bare NOT IN: a null type code would make the predicate -- null and drop the row silently. Unknown type is not grounds for exclusion. and coalesce(s.school_type_code, -1) not in ({{ var('non_england_school_type_codes') | join(', ') }}) +-- Welsh schools GIAS does not type as Welsh establishments (see vars). +and coalesce(s.local_authority_code, -1) not between {{ var('welsh_la_code_min') }} and {{ var('welsh_la_code_max') }} diff --git a/pipeline/transform/models/marts/dim_school.sql b/pipeline/transform/models/marts/dim_school.sql index bcb246c..fe7702d 100644 --- a/pipeline/transform/models/marts/dim_school.sql +++ b/pipeline/transform/models/marts/dim_school.sql @@ -96,3 +96,5 @@ where s.status_code in (1, 3) -- coalesce, not a bare NOT IN: a null type code would make the predicate -- null and drop the row silently. Unknown type is not grounds for exclusion. and coalesce(s.school_type_code, -1) not in ({{ var('non_england_school_type_codes') | join(', ') }}) +-- Welsh schools GIAS does not type as Welsh establishments (see vars). +and coalesce(s.local_authority_code, -1) not between {{ var('welsh_la_code_min') }} and {{ var('welsh_la_code_max') }} diff --git a/pipeline/transform/tests/assert_england_only_schools.sql b/pipeline/transform/tests/assert_england_only_schools.sql index 0356e29..05fcd1b 100644 --- a/pipeline/transform/tests/assert_england_only_schools.sql +++ b/pipeline/transform/tests/assert_england_only_schools.sql @@ -26,3 +26,29 @@ select from {{ ref('dim_location') }} l left join {{ ref('dim_school') }} s on l.urn = s.urn where s.urn is null + +union all + +-- Belt and braces on the Welsh LA code range. +-- +-- The range 660-681 is documented GIAS structure, not something this project +-- can verify from its own data — local_authority_code is not exposed anywhere +-- the app can assert on. So the range does the filtering and the authority +-- NAME does the checking: if 660-681 is ever wrong or incomplete, a Welsh +-- authority reappears here and the pipeline fails loudly rather than the +-- corpus quietly regrowing a Welsh tail. +-- +-- Beechwood College (142458) is why this exists: a Welsh school GIAS types as +-- a Special post 16 institution, which the type filter alone let through. +select + l.urn, + l.local_authority_name as school_name, + null::integer as school_type_code +from {{ ref('dim_location') }} l +where l.local_authority_name in ( + 'Isle of Anglesey', 'Gwynedd', 'Conwy', 'Denbighshire', 'Flintshire', + 'Wrexham', 'Powys', 'Ceredigion', 'Pembrokeshire', 'Carmarthenshire', + 'Swansea', 'Neath Port Talbot', 'Bridgend', 'Vale of Glamorgan', + 'Rhondda Cynon Taf', 'Merthyr Tydfil', 'Caerphilly', 'Blaenau Gwent', + 'Torfaen', 'Monmouthshire', 'Newport', 'Cardiff' +)