feat: GIAS code->name dictionaries generated from live bulk CSV

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/gias_codes.py

@@ -0,0 +1,152 @@
+"""GIAS code -> name dictionaries.
+
+GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
+— do not edit by hand; rerun the script when the dbt drift test warns.
+The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
+must be byte-identical (enforced by backend/tests/test_gias_codes.py).
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+
+logger = logging.getLogger(__name__)
+
+
+SCHOOL_TYPE: dict[int, str] = {
+    1: "Community school",
+    2: "Voluntary aided school",
+    3: "Voluntary controlled school",
+    5: "Foundation school",
+    6: "City technology college",
+    7: "Community special school",
+    8: "Non-maintained special school",
+    10: "Other independent special school",
+    11: "Other independent school",
+    12: "Foundation special school",
+    14: "Pupil referral unit",
+    15: "Local authority nursery school",
+    18: "Further education",
+    24: "Secure units",
+    25: "Offshore schools",
+    26: "Service children's education",
+    27: "Miscellaneous",
+    28: "Academy sponsor led",
+    29: "Higher education institutions",
+    30: "Welsh establishment",
+    31: "Sixth form centres",
+    32: "Special post 16 institution",
+    33: "Academy special sponsor led",
+    34: "Academy converter",
+    35: "Free schools",
+    36: "Free schools special",
+    37: "British schools overseas",
+    38: "Free schools alternative provision",
+    39: "Free schools 16 to 19",
+    40: "University technical college",
+    41: "Studio schools",
+    42: "Academy alternative provision converter",
+    43: "Academy alternative provision sponsor led",
+    44: "Academy special converter",
+    45: "Academy 16-19 converter",
+    46: "Academy 16 to 19 sponsor led",
+    49: "Online provider",
+    56: "Institution funded by other government department",
+    57: "Academy secure 16 to 19",
+}
+
+ESTABLISHMENT_STATUS: dict[int, str] = {
+    1: "Open",
+    2: "Closed",
+    3: "Open, but proposed to close",
+    4: "Proposed to open",
+}
+
+PHASE_OF_EDUCATION: dict[int, str] = {
+    0: "Not applicable",
+    1: "Nursery",
+    2: "Primary",
+    3: "Middle deemed primary",
+    4: "Secondary",
+    5: "Middle deemed secondary",
+    6: "16 plus",
+    7: "All-through",
+}
+
+OFFICIAL_SIXTH_FORM: dict[int, str] = {
+    0: "Not applicable",
+    1: "Has a sixth form",
+    2: "Does not have a sixth form",
+}
+
+RELIGIOUS_CHARACTER: dict[int, str] = {
+    0: "Does not apply",
+    2: "Church of England",
+    3: "Roman Catholic",
+    4: "Methodist",
+    5: "Jewish",
+    6: "None",
+    7: "Muslim",
+    8: "Seventh Day Adventist",
+    9: "Church of England/Methodist",
+    10: "Methodist/Church of England",
+    11: "Church of England/Roman Catholic",
+    12: "Church of England/United Reformed Church",
+    13: "Roman Catholic/Church of England",
+    14: "Quaker",
+    15: "Christian",
+    16: "United Reformed Church",
+    17: "Congregational Church",
+    18: "Free Church",
+    19: "Church of England/Free Church",
+    20: "Church of England/Christian",
+    21: "Sikh",
+    22: "Greek Orthodox",
+    24: "Buddhist",
+    25: "Hindu",
+    26: "Moravian",
+    28: "Inter- / non- denominational",
+    29: "Multi-faith",
+    30: "Church of England/Methodist/United Reform Church/Baptist",
+    31: "Anglican",
+    32: "Anglican/Christian",
+    33: "Anglican/Evangelical",
+    34: "Anglican/Church of England",
+    35: "Catholic",
+    36: "Charadi Jewish",
+    37: "Christian/Evangelical",
+    38: "Christian Science",
+    39: "Christian/Methodist",
+    40: "Christian/non-denominational",
+    41: "Church of England/Evangelical",
+    42: "Islam",
+    43: "Orthodox Jewish",
+    44: "Plymouth Brethren Christian Church",
+    45: "Protestant",
+    46: "Protestant/Evangelical",
+    47: "Reformed Baptist",
+    48: "Roman Catholic/Anglican",
+    49: "Sunni Deobandi",
+}
+
+ADMISSIONS_POLICY: dict[int, str] = {
+    0: "Not applicable",
+    2: "Selective",
+    4: "Non-selective",
+}
+
+
+def translate(code, mapping: dict[int, str]) -> str | None:
+    """Translate a GIAS code to its display name.
+
+    None/NaN -> None (column absent or suppressed). Unknown codes degrade to
+    "Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
+    """
+    if code is None or (isinstance(code, float) and math.isnan(code)):
+        return None
+    code = int(code)
+    if code not in mapping:
+        logger.warning("Unknown GIAS code %s (not in dictionary)", code)
+        return f"Unknown ({code})"
+    return mapping[code]

backend/tests/test_gias_codes.py

@@ -0,0 +1,83 @@
+"""Tests for the GIAS code->name dictionaries (spec 2026-07-09).
+
+The dictionaries are generated from the live GIAS bulk CSV by
+pipeline/scripts/generate_gias_codes.py — these tests assert the module's
+contract, key sentinel values the marts/UI depend on, and that the pipeline
+copy has not drifted from the canonical backend module.
+"""
+
+import math
+from pathlib import Path
+
+from backend.gias_codes import (
+    ADMISSIONS_POLICY,
+    ESTABLISHMENT_STATUS,
+    OFFICIAL_SIXTH_FORM,
+    PHASE_OF_EDUCATION,
+    RELIGIOUS_CHARACTER,
+    SCHOOL_TYPE,
+    translate,
+)
+
+REPO = Path(__file__).resolve().parents[2]
+
+
+def test_translate_known_code():
+    open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open")
+    assert translate(open_code, ESTABLISHMENT_STATUS) == "Open"
+
+
+def test_translate_unknown_code_degrades_gracefully():
+    assert translate(9999, ESTABLISHMENT_STATUS) == "Unknown (9999)"
+
+
+def test_translate_none_and_nan_return_none():
+    assert translate(None, ESTABLISHMENT_STATUS) is None
+    assert translate(float("nan"), ESTABLISHMENT_STATUS) is None
+
+
+def test_translate_accepts_float_codes():
+    # pd.read_sql yields float columns when NULLs are present
+    open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open")
+    assert translate(float(open_code), ESTABLISHMENT_STATUS) == "Open"
+
+
+def test_sentinel_names_present():
+    """Names the marts/UI compare against must exist verbatim."""
+    assert "Open" in ESTABLISHMENT_STATUS.values()
+    assert "Open, but proposed to close" in ESTABLISHMENT_STATUS.values()
+    assert "Has a sixth form" in OFFICIAL_SIXTH_FORM.values()
+    assert "Primary" in PHASE_OF_EDUCATION.values()
+    assert "Secondary" in PHASE_OF_EDUCATION.values()
+    assert "Does not apply" in RELIGIOUS_CHARACTER.values()
+    assert all(len(d) > 0 for d in (
+        SCHOOL_TYPE, ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION,
+        OFFICIAL_SIXTH_FORM, RELIGIOUS_CHARACTER, ADMISSIONS_POLICY,
+    ))
+
+
+def test_pipeline_copy_is_identical():
+    canonical = (REPO / "backend" / "gias_codes.py").read_text()
+    copy = (REPO / "pipeline" / "scripts" / "gias_codes.py").read_text()
+    assert canonical == copy, (
+        "pipeline/scripts/gias_codes.py has drifted from backend/gias_codes.py — "
+        "regenerate with pipeline/scripts/generate_gias_codes.py and copy the file"
+    )
+
+
+def test_seed_matches_dictionaries():
+    import csv
+    fields = {
+        "school_type": SCHOOL_TYPE,
+        "establishment_status": ESTABLISHMENT_STATUS,
+        "phase_of_education": PHASE_OF_EDUCATION,
+        "official_sixth_form": OFFICIAL_SIXTH_FORM,
+        "religious_character": RELIGIOUS_CHARACTER,
+        "admissions_policy": ADMISSIONS_POLICY,
+    }
+    seed_path = REPO / "pipeline" / "transform" / "seeds" / "gias_code_names.csv"
+    seed: dict[str, dict[int, str]] = {k: {} for k in fields}
+    with open(seed_path, newline="") as fh:
+        for row in csv.DictReader(fh):
+            seed[row["field"]][int(row["code"])] = row["name"]
+    assert seed == fields

pipeline/scripts/generate_gias_codes.py

@@ -0,0 +1,134 @@
+"""Generate GIAS code->name dictionaries from the live bulk CSV.
+
+Writes:
+  - backend/gias_codes.py                        (canonical Python module)
+  - pipeline/scripts/gias_codes.py               (byte-identical copy)
+  - pipeline/transform/seeds/gias_code_names.csv (dbt seed for drift test)
+
+Run from the repo root whenever the dbt drift test warns that DfE
+added/renamed a value:  python pipeline/scripts/generate_gias_codes.py
+"""
+
+from __future__ import annotations
+
+import io
+import sys
+from datetime import date, timedelta
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+GIAS_URL = (
+    "https://ea-edubase-api-prod.azurewebsites.net"
+    "/edubase/downloads/public/edubasealldata{date}.csv"
+)
+
+# (CSV code column, CSV name column, python dict name, seed field key)
+FIELDS = [
+    ("TypeOfEstablishment (code)", "TypeOfEstablishment (name)", "SCHOOL_TYPE", "school_type"),
+    ("EstablishmentStatus (code)", "EstablishmentStatus (name)", "ESTABLISHMENT_STATUS", "establishment_status"),
+    ("PhaseOfEducation (code)", "PhaseOfEducation (name)", "PHASE_OF_EDUCATION", "phase_of_education"),
+    ("OfficialSixthForm (code)", "OfficialSixthForm (name)", "OFFICIAL_SIXTH_FORM", "official_sixth_form"),
+    ("ReligiousCharacter (code)", "ReligiousCharacter (name)", "RELIGIOUS_CHARACTER", "religious_character"),
+    ("AdmissionsPolicy (code)", "AdmissionsPolicy (name)", "ADMISSIONS_POLICY", "admissions_policy"),
+]
+
+MODULE_HEADER = '''"""GIAS code -> name dictionaries.
+
+GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
+— do not edit by hand; rerun the script when the dbt drift test warns.
+The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
+must be byte-identical (enforced by backend/tests/test_gias_codes.py).
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+
+logger = logging.getLogger(__name__)
+
+'''
+
+MODULE_FOOTER = '''
+
+def translate(code, mapping: dict[int, str]) -> str | None:
+    """Translate a GIAS code to its display name.
+
+    None/NaN -> None (column absent or suppressed). Unknown codes degrade to
+    "Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
+    """
+    if code is None or (isinstance(code, float) and math.isnan(code)):
+        return None
+    code = int(code)
+    if code not in mapping:
+        logger.warning("Unknown GIAS code %s (not in dictionary)", code)
+        return f"Unknown ({code})"
+    return mapping[code]
+'''
+
+
+def download_csv() -> pd.DataFrame:
+    for day in (date.today(), date.today() - timedelta(days=1)):
+        url = GIAS_URL.format(date=day.strftime("%Y%m%d"))
+        print(f"Downloading {url}")
+        resp = requests.get(url, timeout=300)
+        if resp.status_code == 404:
+            continue
+        resp.raise_for_status()
+        return pd.read_csv(
+            io.StringIO(resp.content.decode("latin-1")),
+            dtype=str, keep_default_na=False,
+        )
+    sys.exit("GIAS CSV not available for today or yesterday")
+
+
+def main() -> None:
+    repo = Path(__file__).resolve().parents[2]
+    df = download_csv()
+
+    module_parts = [MODULE_HEADER]
+    seed_rows: list[tuple[str, int, str]] = []
+
+    for code_col, name_col, dict_name, field_key in FIELDS:
+        pairs = (
+            df[[code_col, name_col]]
+            .loc[lambda d: (d[code_col] != "") & (d[name_col] != "")]
+            .drop_duplicates()
+        )
+        mapping = sorted((int(c), n) for c, n in pairs.itertuples(index=False))
+        dupes = len(mapping) - len({c for c, _ in mapping})
+        if dupes:
+            sys.exit(f"{code_col}: {dupes} codes map to multiple names — investigate before generating")
+        lines = [f"{dict_name}: dict[int, str] = {{"]
+        for code, name in mapping:
+            escaped = name.replace('"', '\\"')
+            lines.append(f'    {code}: "{escaped}",')
+        lines.append("}\n")
+        module_parts.append("\n".join(lines))
+        seed_rows += [(field_key, code, name) for code, name in mapping]
+
+    module = "\n".join(module_parts) + MODULE_FOOTER
+
+    (repo / "backend" / "gias_codes.py").write_text(module)
+    (repo / "pipeline" / "scripts" / "gias_codes.py").write_text(module)
+
+    seed_path = repo / "pipeline" / "transform" / "seeds" / "gias_code_names.csv"
+    with open(seed_path, "w", newline="") as fh:
+        import csv
+        w = csv.writer(fh)
+        w.writerow(["field", "code", "name"])
+        w.writerows(seed_rows)
+
+    print(f"Wrote backend/gias_codes.py, pipeline/scripts/gias_codes.py, {seed_path.name}")
+    print("\nKey codes for the dbt work (Task 3):")
+    for field in ("establishment_status", "phase_of_education", "official_sixth_form"):
+        print(f"  {field}:")
+        for f, code, name in seed_rows:
+            if f == field:
+                print(f"    {code} = {name}")
+
+
+if __name__ == "__main__":
+    main()

pipeline/scripts/gias_codes.py

@@ -0,0 +1,152 @@
+"""GIAS code -> name dictionaries.
+
+GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
+— do not edit by hand; rerun the script when the dbt drift test warns.
+The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
+must be byte-identical (enforced by backend/tests/test_gias_codes.py).
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+
+logger = logging.getLogger(__name__)
+
+
+SCHOOL_TYPE: dict[int, str] = {
+    1: "Community school",
+    2: "Voluntary aided school",
+    3: "Voluntary controlled school",
+    5: "Foundation school",
+    6: "City technology college",
+    7: "Community special school",
+    8: "Non-maintained special school",
+    10: "Other independent special school",
+    11: "Other independent school",
+    12: "Foundation special school",
+    14: "Pupil referral unit",
+    15: "Local authority nursery school",
+    18: "Further education",
+    24: "Secure units",
+    25: "Offshore schools",
+    26: "Service children's education",
+    27: "Miscellaneous",
+    28: "Academy sponsor led",
+    29: "Higher education institutions",
+    30: "Welsh establishment",
+    31: "Sixth form centres",
+    32: "Special post 16 institution",
+    33: "Academy special sponsor led",
+    34: "Academy converter",
+    35: "Free schools",
+    36: "Free schools special",
+    37: "British schools overseas",
+    38: "Free schools alternative provision",
+    39: "Free schools 16 to 19",
+    40: "University technical college",
+    41: "Studio schools",
+    42: "Academy alternative provision converter",
+    43: "Academy alternative provision sponsor led",
+    44: "Academy special converter",
+    45: "Academy 16-19 converter",
+    46: "Academy 16 to 19 sponsor led",
+    49: "Online provider",
+    56: "Institution funded by other government department",
+    57: "Academy secure 16 to 19",
+}
+
+ESTABLISHMENT_STATUS: dict[int, str] = {
+    1: "Open",
+    2: "Closed",
+    3: "Open, but proposed to close",
+    4: "Proposed to open",
+}
+
+PHASE_OF_EDUCATION: dict[int, str] = {
+    0: "Not applicable",
+    1: "Nursery",
+    2: "Primary",
+    3: "Middle deemed primary",
+    4: "Secondary",
+    5: "Middle deemed secondary",
+    6: "16 plus",
+    7: "All-through",
+}
+
+OFFICIAL_SIXTH_FORM: dict[int, str] = {
+    0: "Not applicable",
+    1: "Has a sixth form",
+    2: "Does not have a sixth form",
+}
+
+RELIGIOUS_CHARACTER: dict[int, str] = {
+    0: "Does not apply",
+    2: "Church of England",
+    3: "Roman Catholic",
+    4: "Methodist",
+    5: "Jewish",
+    6: "None",
+    7: "Muslim",
+    8: "Seventh Day Adventist",
+    9: "Church of England/Methodist",
+    10: "Methodist/Church of England",
+    11: "Church of England/Roman Catholic",
+    12: "Church of England/United Reformed Church",
+    13: "Roman Catholic/Church of England",
+    14: "Quaker",
+    15: "Christian",
+    16: "United Reformed Church",
+    17: "Congregational Church",
+    18: "Free Church",
+    19: "Church of England/Free Church",
+    20: "Church of England/Christian",
+    21: "Sikh",
+    22: "Greek Orthodox",
+    24: "Buddhist",
+    25: "Hindu",
+    26: "Moravian",
+    28: "Inter- / non- denominational",
+    29: "Multi-faith",
+    30: "Church of England/Methodist/United Reform Church/Baptist",
+    31: "Anglican",
+    32: "Anglican/Christian",
+    33: "Anglican/Evangelical",
+    34: "Anglican/Church of England",
+    35: "Catholic",
+    36: "Charadi Jewish",
+    37: "Christian/Evangelical",
+    38: "Christian Science",
+    39: "Christian/Methodist",
+    40: "Christian/non-denominational",
+    41: "Church of England/Evangelical",
+    42: "Islam",
+    43: "Orthodox Jewish",
+    44: "Plymouth Brethren Christian Church",
+    45: "Protestant",
+    46: "Protestant/Evangelical",
+    47: "Reformed Baptist",
+    48: "Roman Catholic/Anglican",
+    49: "Sunni Deobandi",
+}
+
+ADMISSIONS_POLICY: dict[int, str] = {
+    0: "Not applicable",
+    2: "Selective",
+    4: "Non-selective",
+}
+
+
+def translate(code, mapping: dict[int, str]) -> str | None:
+    """Translate a GIAS code to its display name.
+
+    None/NaN -> None (column absent or suppressed). Unknown codes degrade to
+    "Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
+    """
+    if code is None or (isinstance(code, float) and math.isnan(code)):
+        return None
+    code = int(code)
+    if code not in mapping:
+        logger.warning("Unknown GIAS code %s (not in dictionary)", code)
+        return f"Unknown ({code})"
+    return mapping[code]

pipeline/transform/seeds/gias_code_names.csv

@@ -0,0 +1,105 @@
+field,code,name
+school_type,1,Community school
+school_type,2,Voluntary aided school
+school_type,3,Voluntary controlled school
+school_type,5,Foundation school
+school_type,6,City technology college
+school_type,7,Community special school
+school_type,8,Non-maintained special school
+school_type,10,Other independent special school
+school_type,11,Other independent school
+school_type,12,Foundation special school
+school_type,14,Pupil referral unit
+school_type,15,Local authority nursery school
+school_type,18,Further education
+school_type,24,Secure units
+school_type,25,Offshore schools
+school_type,26,Service children's education
+school_type,27,Miscellaneous
+school_type,28,Academy sponsor led
+school_type,29,Higher education institutions
+school_type,30,Welsh establishment
+school_type,31,Sixth form centres
+school_type,32,Special post 16 institution
+school_type,33,Academy special sponsor led
+school_type,34,Academy converter
+school_type,35,Free schools
+school_type,36,Free schools special
+school_type,37,British schools overseas
+school_type,38,Free schools alternative provision
+school_type,39,Free schools 16 to 19
+school_type,40,University technical college
+school_type,41,Studio schools
+school_type,42,Academy alternative provision converter
+school_type,43,Academy alternative provision sponsor led
+school_type,44,Academy special converter
+school_type,45,Academy 16-19 converter
+school_type,46,Academy 16 to 19 sponsor led
+school_type,49,Online provider
+school_type,56,Institution funded by other government department
+school_type,57,Academy secure 16 to 19
+establishment_status,1,Open
+establishment_status,2,Closed
+establishment_status,3,"Open, but proposed to close"
+establishment_status,4,Proposed to open
+phase_of_education,0,Not applicable
+phase_of_education,1,Nursery
+phase_of_education,2,Primary
+phase_of_education,3,Middle deemed primary
+phase_of_education,4,Secondary
+phase_of_education,5,Middle deemed secondary
+phase_of_education,6,16 plus
+phase_of_education,7,All-through
+official_sixth_form,0,Not applicable
+official_sixth_form,1,Has a sixth form
+official_sixth_form,2,Does not have a sixth form
+religious_character,0,Does not apply
+religious_character,2,Church of England
+religious_character,3,Roman Catholic
+religious_character,4,Methodist
+religious_character,5,Jewish
+religious_character,6,None
+religious_character,7,Muslim
+religious_character,8,Seventh Day Adventist
+religious_character,9,Church of England/Methodist
+religious_character,10,Methodist/Church of England
+religious_character,11,Church of England/Roman Catholic
+religious_character,12,Church of England/United Reformed Church
+religious_character,13,Roman Catholic/Church of England
+religious_character,14,Quaker
+religious_character,15,Christian
+religious_character,16,United Reformed Church
+religious_character,17,Congregational Church
+religious_character,18,Free Church
+religious_character,19,Church of England/Free Church
+religious_character,20,Church of England/Christian
+religious_character,21,Sikh
+religious_character,22,Greek Orthodox
+religious_character,24,Buddhist
+religious_character,25,Hindu
+religious_character,26,Moravian
+religious_character,28,Inter- / non- denominational
+religious_character,29,Multi-faith
+religious_character,30,Church of England/Methodist/United Reform Church/Baptist
+religious_character,31,Anglican
+religious_character,32,Anglican/Christian
+religious_character,33,Anglican/Evangelical
+religious_character,34,Anglican/Church of England
+religious_character,35,Catholic
+religious_character,36,Charadi Jewish
+religious_character,37,Christian/Evangelical
+religious_character,38,Christian Science
+religious_character,39,Christian/Methodist
+religious_character,40,Christian/non-denominational
+religious_character,41,Church of England/Evangelical
+religious_character,42,Islam
+religious_character,43,Orthodox Jewish
+religious_character,44,Plymouth Brethren Christian Church
+religious_character,45,Protestant
+religious_character,46,Protestant/Evangelical
+religious_character,47,Reformed Baptist
+religious_character,48,Roman Catholic/Anglican
+religious_character,49,Sunni Deobandi
+admissions_policy,0,Not applicable
+admissions_policy,2,Selective
+admissions_policy,4,Non-selective