feat(pipeline): dim_school/dim_location store GIAS codes; seed drift test

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tudor
2026-07-09 10:45:15 +01:00
co-authored by Claude Fable 5
parent d898e6279b
commit fa6c929a3a
4 changed files with 78 additions and 32 deletions
@@ -0,0 +1,33 @@
-- Warn when the live GIAS CSV carries a (code, name) pair we don't have in
-- the dictionary seed — i.e. DfE added or renamed a value. Fix by rerunning
-- pipeline/scripts/generate_gias_codes.py and committing the regenerated
-- dictionaries + seed together.
{{ config(severity='warn') }}
with raw_pairs as (
{% for field_key, code_col, name_col in [
('school_type', 'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)'),
('establishment_status', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)'),
('phase_of_education', 'PhaseOfEducation (code)', 'PhaseOfEducation (name)'),
('official_sixth_form', 'OfficialSixthForm (code)', 'OfficialSixthForm (name)'),
('religious_character', 'ReligiousCharacter (code)', 'ReligiousCharacter (name)'),
('admissions_policy', 'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)')
] %}
select distinct
'{{ field_key }}' as field,
cast(nullif(trim("{{ code_col }}"), '') as integer) as code,
nullif(trim("{{ name_col }}"), '') as name
from {{ source('raw', 'gias_establishments') }}
where nullif(trim("{{ code_col }}"), '') is not null
and nullif(trim("{{ name_col }}"), '') is not null
{% if not loop.last %}union all{% endif %}
{% endfor %}
)
select r.*
from raw_pairs r
left join {{ ref('gias_code_names') }} s
on s.field = r.field
and s.code = r.code
and s.name = r.name
where s.field is null