diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 4b89d73..d5dfd26 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -8,9 +8,10 @@ models: tests: [not_null, unique] - name: school_name tests: [not_null] - - name: phase + - name: phase_code description: > - Primary / Secondary / All-through etc. May be null for a small number + GIAS PhaseOfEducation code (2 = Primary, 4 = Secondary, 7 = All-through, + etc. — see seeds/gias_code_names.csv). May be null for a small number of independent schools where GIAS publishes "Not Applicable", no statutory age range, and the school name gives no hint. tests: @@ -27,10 +28,26 @@ models: - not_null - accepted_values: values: [true, false] - - name: status + - name: status_code + description: GIAS EstablishmentStatus code (1 = Open, 3 = Open but proposed to close) tests: - accepted_values: - values: ["Open", "Open, but proposed to close"] + values: [1, 3] + - name: school_type_code + tests: + - accepted_values: + severity: warn + values: [1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 14, 15, 18, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 49, 56, 57] + - name: religious_character_code + tests: + - accepted_values: + severity: warn + values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] + - name: admissions_policy_code + tests: + - accepted_values: + severity: warn + values: [0, 2, 4] - name: dim_location description: School location dimension with PostGIS geometry diff --git a/pipeline/transform/models/marts/dim_location.sql b/pipeline/transform/models/marts/dim_location.sql index 7c13c4b..b05e8a7 100644 --- a/pipeline/transform/models/marts/dim_location.sql +++ b/pipeline/transform/models/marts/dim_location.sql @@ -32,4 +32,4 @@ select end as longitude from {{ ref('stg_gias_establishments') }} s -- Must match dim_school's status filter exactly (the API inner-joins the two). -where s.status in ('Open', 'Open, but proposed to close') +where s.status_code in (1, 3) diff --git a/pipeline/transform/models/marts/dim_school.sql b/pipeline/transform/models/marts/dim_school.sql index fbcc8b1..86dfa2e 100644 --- a/pipeline/transform/models/marts/dim_school.sql +++ b/pipeline/transform/models/marts/dim_school.sql @@ -19,16 +19,17 @@ select s.urn, s.local_authority_code * 1000 + s.establishment_number as laestab, s.school_name, + -- Phase in GIAS code space (see seeds/gias_code_names.csv): + -- 2 = Primary, 4 = Secondary, 7 = All-through, 0 = Not applicable. case - -- 1. Trust GIAS phase when it's a real value (not the catch-all "Not Applicable") - when s.phase is not null - and lower(trim(s.phase)) not in ('not applicable', '', 'unknown') - then s.phase + -- 1. Trust GIAS phase when it's a real value (0 = the catch-all "Not Applicable") + when s.phase_code is not null and s.phase_code != 0 + then s.phase_code -- 2. Infer from statutory age range (independent schools still publish these) - when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 'Primary' - when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 'Secondary' + when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 2 + when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 4 when s.statutory_low_age is not null and s.statutory_high_age is not null - and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 'All-through' + and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 7 -- 3. Fallback: infer from school name (covers independents with missing ages) when s.school_name ilike '%primary%' or s.school_name ilike '%infant%' @@ -36,31 +37,27 @@ select or s.school_name ilike '%preparatory%' or s.school_name ilike '% prep school%' or s.school_name ilike '% prep %' - then 'Primary' + then 2 when s.school_name ilike '%secondary%' or s.school_name ilike '%high school%' or s.school_name ilike '%grammar%' or s.school_name ilike '%senior school%' or s.school_name ilike '%upper school%' - then 'Secondary' - -- 4. Give up — leave phase null so the UI renders no pill + then 4 + -- 4. Give up — null renders no phase pill else null - end as phase, - s.school_type, + end as phase_code, + s.school_type_code, s.academy_trust_name, s.academy_trust_uid, - s.religious_character, + s.religious_character_code, s.gender, s.statutory_low_age || '-' || s.statutory_high_age as age_range, - -- Authoritative sixth-form flag (spec §3): GIAS OfficialSixthForm. - -- "Not applicable" (nurseries, primaries, PRUs) => false. Blank GIAS - -- value (rare, new establishments) falls back to the statutory age range. - -- lower(trim()) guards against casing/whitespace variants in raw GIAS - -- data, same as the phase derivation above — an unmatched variant would - -- otherwise silently fall through to the age-range fallback. + -- GIAS OfficialSixthForm in code space: 1 = has, 2 = does not, 0 = N/A. + -- Null (rare, new establishments) falls back to the statutory age range. case - when lower(trim(s.official_sixth_form)) = 'has a sixth form' then true - when lower(trim(s.official_sixth_form)) in ('does not have a sixth form', 'not applicable') then false + when s.official_sixth_form_code = 1 then true + when s.official_sixth_form_code in (0, 2) then false else coalesce(s.statutory_high_age >= 18, false) end as has_sixth_form, s.capacity, @@ -70,9 +67,9 @@ select s.telephone, s.open_date, s.close_date, - s.status, + s.status_code, s.nursery_provision, - s.admissions_policy, + s.admissions_policy_code, -- Latest Ofsted (populated after monthly Ofsted pipeline runs) {% if ofsted_relation is not none %} @@ -91,7 +88,6 @@ from schools s {% if ofsted_relation is not none %} left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn {% endif %} --- "Open, but proposed to close" schools are still operating (pupils enrolled, --- results published) — include them; they drop out automatically once GIAS --- flips them to "Closed" (marts are fully rebuilt each run). -where s.status in ('Open', 'Open, but proposed to close') +-- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when +-- GIAS flips to Closed — marts fully rebuild each run). +where s.status_code in (1, 3) diff --git a/pipeline/transform/tests/assert_gias_code_names_match_seed.sql b/pipeline/transform/tests/assert_gias_code_names_match_seed.sql new file mode 100644 index 0000000..b3e9ef1 --- /dev/null +++ b/pipeline/transform/tests/assert_gias_code_names_match_seed.sql @@ -0,0 +1,33 @@ +-- Warn when the live GIAS CSV carries a (code, name) pair we don't have in +-- the dictionary seed — i.e. DfE added or renamed a value. Fix by rerunning +-- pipeline/scripts/generate_gias_codes.py and committing the regenerated +-- dictionaries + seed together. +{{ config(severity='warn') }} + +with raw_pairs as ( + {% for field_key, code_col, name_col in [ + ('school_type', 'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)'), + ('establishment_status', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)'), + ('phase_of_education', 'PhaseOfEducation (code)', 'PhaseOfEducation (name)'), + ('official_sixth_form', 'OfficialSixthForm (code)', 'OfficialSixthForm (name)'), + ('religious_character', 'ReligiousCharacter (code)', 'ReligiousCharacter (name)'), + ('admissions_policy', 'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)') + ] %} + select distinct + '{{ field_key }}' as field, + cast(nullif(trim("{{ code_col }}"), '') as integer) as code, + nullif(trim("{{ name_col }}"), '') as name + from {{ source('raw', 'gias_establishments') }} + where nullif(trim("{{ code_col }}"), '') is not null + and nullif(trim("{{ name_col }}"), '') is not null + {% if not loop.last %}union all{% endif %} + {% endfor %} +) + +select r.* +from raw_pairs r +left join {{ ref('gias_code_names') }} s + on s.field = r.field + and s.code = r.code + and s.name = r.name +where s.field is null