feat(pipeline): dim_school/dim_location store GIAS codes; seed drift test
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
-- Warn when the live GIAS CSV carries a (code, name) pair we don't have in
|
||||
-- the dictionary seed — i.e. DfE added or renamed a value. Fix by rerunning
|
||||
-- pipeline/scripts/generate_gias_codes.py and committing the regenerated
|
||||
-- dictionaries + seed together.
|
||||
{{ config(severity='warn') }}
|
||||
|
||||
with raw_pairs as (
|
||||
{% for field_key, code_col, name_col in [
|
||||
('school_type', 'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)'),
|
||||
('establishment_status', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)'),
|
||||
('phase_of_education', 'PhaseOfEducation (code)', 'PhaseOfEducation (name)'),
|
||||
('official_sixth_form', 'OfficialSixthForm (code)', 'OfficialSixthForm (name)'),
|
||||
('religious_character', 'ReligiousCharacter (code)', 'ReligiousCharacter (name)'),
|
||||
('admissions_policy', 'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)')
|
||||
] %}
|
||||
select distinct
|
||||
'{{ field_key }}' as field,
|
||||
cast(nullif(trim("{{ code_col }}"), '') as integer) as code,
|
||||
nullif(trim("{{ name_col }}"), '') as name
|
||||
from {{ source('raw', 'gias_establishments') }}
|
||||
where nullif(trim("{{ code_col }}"), '') is not null
|
||||
and nullif(trim("{{ name_col }}"), '') is not null
|
||||
{% if not loop.last %}union all{% endif %}
|
||||
{% endfor %}
|
||||
)
|
||||
|
||||
select r.*
|
||||
from raw_pairs r
|
||||
left join {{ ref('gias_code_names') }} s
|
||||
on s.field = r.field
|
||||
and s.code = r.code
|
||||
and s.name = r.name
|
||||
where s.field is null
|
||||
Reference in New Issue
Block a user