feat(pipeline): dim_school/dim_location store GIAS codes; seed drift test

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

pipeline/transform/models/marts/_marts_schema.yml

@@ -8,9 +8,10 @@ models:
         tests: [not_null, unique]
       - name: school_name
         tests: [not_null]
-      - name: phase
+      - name: phase_code
         description: >
-          Primary / Secondary / All-through etc. May be null for a small number
+          GIAS PhaseOfEducation code (2 = Primary, 4 = Secondary, 7 = All-through,
+          etc. — see seeds/gias_code_names.csv). May be null for a small number
           of independent schools where GIAS publishes "Not Applicable", no
           statutory age range, and the school name gives no hint.
         tests:
@@ -27,10 +28,26 @@ models:
           - not_null
           - accepted_values:
               values: [true, false]
-      - name: status
+      - name: status_code
+        description: GIAS EstablishmentStatus code (1 = Open, 3 = Open but proposed to close)
         tests:
           - accepted_values:
-              values: ["Open", "Open, but proposed to close"]
+              values: [1, 3]
+      - name: school_type_code
+        tests:
+          - accepted_values:
+              severity: warn
+              values: [1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 14, 15, 18, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 49, 56, 57]
+      - name: religious_character_code
+        tests:
+          - accepted_values:
+              severity: warn
+              values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
+      - name: admissions_policy_code
+        tests:
+          - accepted_values:
+              severity: warn
+              values: [0, 2, 4]
 
   - name: dim_location
     description: School location dimension with PostGIS geometry

pipeline/transform/models/marts/dim_location.sql

@@ -32,4 +32,4 @@ select
     end as longitude
 from {{ ref('stg_gias_establishments') }} s
 -- Must match dim_school's status filter exactly (the API inner-joins the two).
-where s.status in ('Open', 'Open, but proposed to close')
+where s.status_code in (1, 3)

pipeline/transform/models/marts/dim_school.sql

@@ -19,16 +19,17 @@ select
     s.urn,
     s.local_authority_code * 1000 + s.establishment_number as laestab,
     s.school_name,
+    -- Phase in GIAS code space (see seeds/gias_code_names.csv):
+    -- 2 = Primary, 4 = Secondary, 7 = All-through, 0 = Not applicable.
     case
-        -- 1. Trust GIAS phase when it's a real value (not the catch-all "Not Applicable")
+        -- 1. Trust GIAS phase when it's a real value (0 = the catch-all "Not Applicable")
-        when s.phase is not null
+        when s.phase_code is not null and s.phase_code != 0
-             and lower(trim(s.phase)) not in ('not applicable', '', 'unknown')
+            then s.phase_code
-            then s.phase
         -- 2. Infer from statutory age range (independent schools still publish these)
-        when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 'Primary'
+        when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 2
-        when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 'Secondary'
+        when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 4
         when s.statutory_low_age is not null and s.statutory_high_age is not null
-             and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 'All-through'
+             and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 7
         -- 3. Fallback: infer from school name (covers independents with missing ages)
         when s.school_name ilike '%primary%'
              or s.school_name ilike '%infant%'
@@ -36,31 +37,27 @@ select
              or s.school_name ilike '%preparatory%'
              or s.school_name ilike '% prep school%'
              or s.school_name ilike '% prep %'
-            then 'Primary'
+            then 2
         when s.school_name ilike '%secondary%'
              or s.school_name ilike '%high school%'
              or s.school_name ilike '%grammar%'
              or s.school_name ilike '%senior school%'
              or s.school_name ilike '%upper school%'
-            then 'Secondary'
+            then 4
-        -- 4. Give up — leave phase null so the UI renders no pill
+        -- 4. Give up — null renders no phase pill
         else null
-    end as phase,
+    end as phase_code,
-    s.school_type,
+    s.school_type_code,
     s.academy_trust_name,
     s.academy_trust_uid,
-    s.religious_character,
+    s.religious_character_code,
     s.gender,
     s.statutory_low_age || '-' || s.statutory_high_age as age_range,
-    -- Authoritative sixth-form flag (spec §3): GIAS OfficialSixthForm.
+    -- GIAS OfficialSixthForm in code space: 1 = has, 2 = does not, 0 = N/A.
-    -- "Not applicable" (nurseries, primaries, PRUs) => false. Blank GIAS
+    -- Null (rare, new establishments) falls back to the statutory age range.
-    -- value (rare, new establishments) falls back to the statutory age range.
-    -- lower(trim()) guards against casing/whitespace variants in raw GIAS
-    -- data, same as the phase derivation above — an unmatched variant would
-    -- otherwise silently fall through to the age-range fallback.
     case
-        when lower(trim(s.official_sixth_form)) = 'has a sixth form' then true
+        when s.official_sixth_form_code = 1 then true
-        when lower(trim(s.official_sixth_form)) in ('does not have a sixth form', 'not applicable') then false
+        when s.official_sixth_form_code in (0, 2) then false
         else coalesce(s.statutory_high_age >= 18, false)
     end as has_sixth_form,
     s.capacity,
@@ -70,9 +67,9 @@ select
     s.telephone,
     s.open_date,
     s.close_date,
-    s.status,
+    s.status_code,
     s.nursery_provision,
-    s.admissions_policy,
+    s.admissions_policy_code,
 
     -- Latest Ofsted (populated after monthly Ofsted pipeline runs)
     {% if ofsted_relation is not none %}
@@ -91,7 +88,6 @@ from schools s
 {% if ofsted_relation is not none %}
 left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn
 {% endif %}
--- "Open, but proposed to close" schools are still operating (pupils enrolled,
+-- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when
--- results published) — include them; they drop out automatically once GIAS
+-- GIAS flips to Closed — marts fully rebuild each run).
--- flips them to "Closed" (marts are fully rebuilt each run).
+where s.status_code in (1, 3)
-where s.status in ('Open', 'Open, but proposed to close')

pipeline/transform/tests/assert_gias_code_names_match_seed.sql

@@ -0,0 +1,33 @@
+-- Warn when the live GIAS CSV carries a (code, name) pair we don't have in
+-- the dictionary seed — i.e. DfE added or renamed a value. Fix by rerunning
+-- pipeline/scripts/generate_gias_codes.py and committing the regenerated
+-- dictionaries + seed together.
+{{ config(severity='warn') }}
+
+with raw_pairs as (
+    {% for field_key, code_col, name_col in [
+        ('school_type',          'TypeOfEstablishment (code)',  'TypeOfEstablishment (name)'),
+        ('establishment_status', 'EstablishmentStatus (code)',  'EstablishmentStatus (name)'),
+        ('phase_of_education',   'PhaseOfEducation (code)',     'PhaseOfEducation (name)'),
+        ('official_sixth_form',  'OfficialSixthForm (code)',    'OfficialSixthForm (name)'),
+        ('religious_character',  'ReligiousCharacter (code)',   'ReligiousCharacter (name)'),
+        ('admissions_policy',    'AdmissionsPolicy (code)',     'AdmissionsPolicy (name)')
+    ] %}
+    select distinct
+        '{{ field_key }}' as field,
+        cast(nullif(trim("{{ code_col }}"), '') as integer) as code,
+        nullif(trim("{{ name_col }}"), '') as name
+    from {{ source('raw', 'gias_establishments') }}
+    where nullif(trim("{{ code_col }}"), '') is not null
+      and nullif(trim("{{ name_col }}"), '') is not null
+    {% if not loop.last %}union all{% endif %}
+    {% endfor %}
+)
+
+select r.*
+from raw_pairs r
+left join {{ ref('gias_code_names') }} s
+    on s.field = r.field
+   and s.code = r.code
+   and s.name = r.name
+where s.field is null