From e188c2ff4be36fe84ca4bb915bf5597d6c41ad66 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 9 Jul 2026 10:38:19 +0100 Subject: [PATCH] feat: GIAS code->name dictionaries generated from live bulk CSV Co-Authored-By: Claude Fable 5 --- backend/gias_codes.py | 152 +++++++++++++++++++ backend/tests/test_gias_codes.py | 83 ++++++++++ pipeline/scripts/generate_gias_codes.py | 134 ++++++++++++++++ pipeline/scripts/gias_codes.py | 152 +++++++++++++++++++ pipeline/transform/seeds/gias_code_names.csv | 105 +++++++++++++ 5 files changed, 626 insertions(+) create mode 100644 backend/gias_codes.py create mode 100644 backend/tests/test_gias_codes.py create mode 100644 pipeline/scripts/generate_gias_codes.py create mode 100644 pipeline/scripts/gias_codes.py create mode 100644 pipeline/transform/seeds/gias_code_names.csv diff --git a/backend/gias_codes.py b/backend/gias_codes.py new file mode 100644 index 0000000..e454e8a --- /dev/null +++ b/backend/gias_codes.py @@ -0,0 +1,152 @@ +"""GIAS code -> name dictionaries. + +GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV +— do not edit by hand; rerun the script when the dbt drift test warns. +The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py +must be byte-identical (enforced by backend/tests/test_gias_codes.py). +""" + +from __future__ import annotations + +import logging +import math + +logger = logging.getLogger(__name__) + + +SCHOOL_TYPE: dict[int, str] = { + 1: "Community school", + 2: "Voluntary aided school", + 3: "Voluntary controlled school", + 5: "Foundation school", + 6: "City technology college", + 7: "Community special school", + 8: "Non-maintained special school", + 10: "Other independent special school", + 11: "Other independent school", + 12: "Foundation special school", + 14: "Pupil referral unit", + 15: "Local authority nursery school", + 18: "Further education", + 24: "Secure units", + 25: "Offshore schools", + 26: "Service children's education", + 27: "Miscellaneous", + 28: "Academy sponsor led", + 29: "Higher education institutions", + 30: "Welsh establishment", + 31: "Sixth form centres", + 32: "Special post 16 institution", + 33: "Academy special sponsor led", + 34: "Academy converter", + 35: "Free schools", + 36: "Free schools special", + 37: "British schools overseas", + 38: "Free schools alternative provision", + 39: "Free schools 16 to 19", + 40: "University technical college", + 41: "Studio schools", + 42: "Academy alternative provision converter", + 43: "Academy alternative provision sponsor led", + 44: "Academy special converter", + 45: "Academy 16-19 converter", + 46: "Academy 16 to 19 sponsor led", + 49: "Online provider", + 56: "Institution funded by other government department", + 57: "Academy secure 16 to 19", +} + +ESTABLISHMENT_STATUS: dict[int, str] = { + 1: "Open", + 2: "Closed", + 3: "Open, but proposed to close", + 4: "Proposed to open", +} + +PHASE_OF_EDUCATION: dict[int, str] = { + 0: "Not applicable", + 1: "Nursery", + 2: "Primary", + 3: "Middle deemed primary", + 4: "Secondary", + 5: "Middle deemed secondary", + 6: "16 plus", + 7: "All-through", +} + +OFFICIAL_SIXTH_FORM: dict[int, str] = { + 0: "Not applicable", + 1: "Has a sixth form", + 2: "Does not have a sixth form", +} + +RELIGIOUS_CHARACTER: dict[int, str] = { + 0: "Does not apply", + 2: "Church of England", + 3: "Roman Catholic", + 4: "Methodist", + 5: "Jewish", + 6: "None", + 7: "Muslim", + 8: "Seventh Day Adventist", + 9: "Church of England/Methodist", + 10: "Methodist/Church of England", + 11: "Church of England/Roman Catholic", + 12: "Church of England/United Reformed Church", + 13: "Roman Catholic/Church of England", + 14: "Quaker", + 15: "Christian", + 16: "United Reformed Church", + 17: "Congregational Church", + 18: "Free Church", + 19: "Church of England/Free Church", + 20: "Church of England/Christian", + 21: "Sikh", + 22: "Greek Orthodox", + 24: "Buddhist", + 25: "Hindu", + 26: "Moravian", + 28: "Inter- / non- denominational", + 29: "Multi-faith", + 30: "Church of England/Methodist/United Reform Church/Baptist", + 31: "Anglican", + 32: "Anglican/Christian", + 33: "Anglican/Evangelical", + 34: "Anglican/Church of England", + 35: "Catholic", + 36: "Charadi Jewish", + 37: "Christian/Evangelical", + 38: "Christian Science", + 39: "Christian/Methodist", + 40: "Christian/non-denominational", + 41: "Church of England/Evangelical", + 42: "Islam", + 43: "Orthodox Jewish", + 44: "Plymouth Brethren Christian Church", + 45: "Protestant", + 46: "Protestant/Evangelical", + 47: "Reformed Baptist", + 48: "Roman Catholic/Anglican", + 49: "Sunni Deobandi", +} + +ADMISSIONS_POLICY: dict[int, str] = { + 0: "Not applicable", + 2: "Selective", + 4: "Non-selective", +} + + +def translate(code, mapping: dict[int, str]) -> str | None: + """Translate a GIAS code to its display name. + + None/NaN -> None (column absent or suppressed). Unknown codes degrade to + "Unknown ()" with a warning so a new DfE value never blanks the UI. + """ + if code is None or (isinstance(code, float) and math.isnan(code)): + return None + code = int(code) + if code not in mapping: + logger.warning("Unknown GIAS code %s (not in dictionary)", code) + return f"Unknown ({code})" + return mapping[code] diff --git a/backend/tests/test_gias_codes.py b/backend/tests/test_gias_codes.py new file mode 100644 index 0000000..95d02ed --- /dev/null +++ b/backend/tests/test_gias_codes.py @@ -0,0 +1,83 @@ +"""Tests for the GIAS code->name dictionaries (spec 2026-07-09). + +The dictionaries are generated from the live GIAS bulk CSV by +pipeline/scripts/generate_gias_codes.py — these tests assert the module's +contract, key sentinel values the marts/UI depend on, and that the pipeline +copy has not drifted from the canonical backend module. +""" + +import math +from pathlib import Path + +from backend.gias_codes import ( + ADMISSIONS_POLICY, + ESTABLISHMENT_STATUS, + OFFICIAL_SIXTH_FORM, + PHASE_OF_EDUCATION, + RELIGIOUS_CHARACTER, + SCHOOL_TYPE, + translate, +) + +REPO = Path(__file__).resolve().parents[2] + + +def test_translate_known_code(): + open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open") + assert translate(open_code, ESTABLISHMENT_STATUS) == "Open" + + +def test_translate_unknown_code_degrades_gracefully(): + assert translate(9999, ESTABLISHMENT_STATUS) == "Unknown (9999)" + + +def test_translate_none_and_nan_return_none(): + assert translate(None, ESTABLISHMENT_STATUS) is None + assert translate(float("nan"), ESTABLISHMENT_STATUS) is None + + +def test_translate_accepts_float_codes(): + # pd.read_sql yields float columns when NULLs are present + open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open") + assert translate(float(open_code), ESTABLISHMENT_STATUS) == "Open" + + +def test_sentinel_names_present(): + """Names the marts/UI compare against must exist verbatim.""" + assert "Open" in ESTABLISHMENT_STATUS.values() + assert "Open, but proposed to close" in ESTABLISHMENT_STATUS.values() + assert "Has a sixth form" in OFFICIAL_SIXTH_FORM.values() + assert "Primary" in PHASE_OF_EDUCATION.values() + assert "Secondary" in PHASE_OF_EDUCATION.values() + assert "Does not apply" in RELIGIOUS_CHARACTER.values() + assert all(len(d) > 0 for d in ( + SCHOOL_TYPE, ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION, + OFFICIAL_SIXTH_FORM, RELIGIOUS_CHARACTER, ADMISSIONS_POLICY, + )) + + +def test_pipeline_copy_is_identical(): + canonical = (REPO / "backend" / "gias_codes.py").read_text() + copy = (REPO / "pipeline" / "scripts" / "gias_codes.py").read_text() + assert canonical == copy, ( + "pipeline/scripts/gias_codes.py has drifted from backend/gias_codes.py — " + "regenerate with pipeline/scripts/generate_gias_codes.py and copy the file" + ) + + +def test_seed_matches_dictionaries(): + import csv + fields = { + "school_type": SCHOOL_TYPE, + "establishment_status": ESTABLISHMENT_STATUS, + "phase_of_education": PHASE_OF_EDUCATION, + "official_sixth_form": OFFICIAL_SIXTH_FORM, + "religious_character": RELIGIOUS_CHARACTER, + "admissions_policy": ADMISSIONS_POLICY, + } + seed_path = REPO / "pipeline" / "transform" / "seeds" / "gias_code_names.csv" + seed: dict[str, dict[int, str]] = {k: {} for k in fields} + with open(seed_path, newline="") as fh: + for row in csv.DictReader(fh): + seed[row["field"]][int(row["code"])] = row["name"] + assert seed == fields diff --git a/pipeline/scripts/generate_gias_codes.py b/pipeline/scripts/generate_gias_codes.py new file mode 100644 index 0000000..bd1e135 --- /dev/null +++ b/pipeline/scripts/generate_gias_codes.py @@ -0,0 +1,134 @@ +"""Generate GIAS code->name dictionaries from the live bulk CSV. + +Writes: + - backend/gias_codes.py (canonical Python module) + - pipeline/scripts/gias_codes.py (byte-identical copy) + - pipeline/transform/seeds/gias_code_names.csv (dbt seed for drift test) + +Run from the repo root whenever the dbt drift test warns that DfE +added/renamed a value: python pipeline/scripts/generate_gias_codes.py +""" + +from __future__ import annotations + +import io +import sys +from datetime import date, timedelta +from pathlib import Path + +import pandas as pd +import requests + +GIAS_URL = ( + "https://ea-edubase-api-prod.azurewebsites.net" + "/edubase/downloads/public/edubasealldata{date}.csv" +) + +# (CSV code column, CSV name column, python dict name, seed field key) +FIELDS = [ + ("TypeOfEstablishment (code)", "TypeOfEstablishment (name)", "SCHOOL_TYPE", "school_type"), + ("EstablishmentStatus (code)", "EstablishmentStatus (name)", "ESTABLISHMENT_STATUS", "establishment_status"), + ("PhaseOfEducation (code)", "PhaseOfEducation (name)", "PHASE_OF_EDUCATION", "phase_of_education"), + ("OfficialSixthForm (code)", "OfficialSixthForm (name)", "OFFICIAL_SIXTH_FORM", "official_sixth_form"), + ("ReligiousCharacter (code)", "ReligiousCharacter (name)", "RELIGIOUS_CHARACTER", "religious_character"), + ("AdmissionsPolicy (code)", "AdmissionsPolicy (name)", "ADMISSIONS_POLICY", "admissions_policy"), +] + +MODULE_HEADER = '''"""GIAS code -> name dictionaries. + +GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV +— do not edit by hand; rerun the script when the dbt drift test warns. +The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py +must be byte-identical (enforced by backend/tests/test_gias_codes.py). +""" + +from __future__ import annotations + +import logging +import math + +logger = logging.getLogger(__name__) + +''' + +MODULE_FOOTER = ''' + +def translate(code, mapping: dict[int, str]) -> str | None: + """Translate a GIAS code to its display name. + + None/NaN -> None (column absent or suppressed). Unknown codes degrade to + "Unknown ()" with a warning so a new DfE value never blanks the UI. + """ + if code is None or (isinstance(code, float) and math.isnan(code)): + return None + code = int(code) + if code not in mapping: + logger.warning("Unknown GIAS code %s (not in dictionary)", code) + return f"Unknown ({code})" + return mapping[code] +''' + + +def download_csv() -> pd.DataFrame: + for day in (date.today(), date.today() - timedelta(days=1)): + url = GIAS_URL.format(date=day.strftime("%Y%m%d")) + print(f"Downloading {url}") + resp = requests.get(url, timeout=300) + if resp.status_code == 404: + continue + resp.raise_for_status() + return pd.read_csv( + io.StringIO(resp.content.decode("latin-1")), + dtype=str, keep_default_na=False, + ) + sys.exit("GIAS CSV not available for today or yesterday") + + +def main() -> None: + repo = Path(__file__).resolve().parents[2] + df = download_csv() + + module_parts = [MODULE_HEADER] + seed_rows: list[tuple[str, int, str]] = [] + + for code_col, name_col, dict_name, field_key in FIELDS: + pairs = ( + df[[code_col, name_col]] + .loc[lambda d: (d[code_col] != "") & (d[name_col] != "")] + .drop_duplicates() + ) + mapping = sorted((int(c), n) for c, n in pairs.itertuples(index=False)) + dupes = len(mapping) - len({c for c, _ in mapping}) + if dupes: + sys.exit(f"{code_col}: {dupes} codes map to multiple names — investigate before generating") + lines = [f"{dict_name}: dict[int, str] = {{"] + for code, name in mapping: + escaped = name.replace('"', '\\"') + lines.append(f' {code}: "{escaped}",') + lines.append("}\n") + module_parts.append("\n".join(lines)) + seed_rows += [(field_key, code, name) for code, name in mapping] + + module = "\n".join(module_parts) + MODULE_FOOTER + + (repo / "backend" / "gias_codes.py").write_text(module) + (repo / "pipeline" / "scripts" / "gias_codes.py").write_text(module) + + seed_path = repo / "pipeline" / "transform" / "seeds" / "gias_code_names.csv" + with open(seed_path, "w", newline="") as fh: + import csv + w = csv.writer(fh) + w.writerow(["field", "code", "name"]) + w.writerows(seed_rows) + + print(f"Wrote backend/gias_codes.py, pipeline/scripts/gias_codes.py, {seed_path.name}") + print("\nKey codes for the dbt work (Task 3):") + for field in ("establishment_status", "phase_of_education", "official_sixth_form"): + print(f" {field}:") + for f, code, name in seed_rows: + if f == field: + print(f" {code} = {name}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/scripts/gias_codes.py b/pipeline/scripts/gias_codes.py new file mode 100644 index 0000000..e454e8a --- /dev/null +++ b/pipeline/scripts/gias_codes.py @@ -0,0 +1,152 @@ +"""GIAS code -> name dictionaries. + +GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV +— do not edit by hand; rerun the script when the dbt drift test warns. +The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py +must be byte-identical (enforced by backend/tests/test_gias_codes.py). +""" + +from __future__ import annotations + +import logging +import math + +logger = logging.getLogger(__name__) + + +SCHOOL_TYPE: dict[int, str] = { + 1: "Community school", + 2: "Voluntary aided school", + 3: "Voluntary controlled school", + 5: "Foundation school", + 6: "City technology college", + 7: "Community special school", + 8: "Non-maintained special school", + 10: "Other independent special school", + 11: "Other independent school", + 12: "Foundation special school", + 14: "Pupil referral unit", + 15: "Local authority nursery school", + 18: "Further education", + 24: "Secure units", + 25: "Offshore schools", + 26: "Service children's education", + 27: "Miscellaneous", + 28: "Academy sponsor led", + 29: "Higher education institutions", + 30: "Welsh establishment", + 31: "Sixth form centres", + 32: "Special post 16 institution", + 33: "Academy special sponsor led", + 34: "Academy converter", + 35: "Free schools", + 36: "Free schools special", + 37: "British schools overseas", + 38: "Free schools alternative provision", + 39: "Free schools 16 to 19", + 40: "University technical college", + 41: "Studio schools", + 42: "Academy alternative provision converter", + 43: "Academy alternative provision sponsor led", + 44: "Academy special converter", + 45: "Academy 16-19 converter", + 46: "Academy 16 to 19 sponsor led", + 49: "Online provider", + 56: "Institution funded by other government department", + 57: "Academy secure 16 to 19", +} + +ESTABLISHMENT_STATUS: dict[int, str] = { + 1: "Open", + 2: "Closed", + 3: "Open, but proposed to close", + 4: "Proposed to open", +} + +PHASE_OF_EDUCATION: dict[int, str] = { + 0: "Not applicable", + 1: "Nursery", + 2: "Primary", + 3: "Middle deemed primary", + 4: "Secondary", + 5: "Middle deemed secondary", + 6: "16 plus", + 7: "All-through", +} + +OFFICIAL_SIXTH_FORM: dict[int, str] = { + 0: "Not applicable", + 1: "Has a sixth form", + 2: "Does not have a sixth form", +} + +RELIGIOUS_CHARACTER: dict[int, str] = { + 0: "Does not apply", + 2: "Church of England", + 3: "Roman Catholic", + 4: "Methodist", + 5: "Jewish", + 6: "None", + 7: "Muslim", + 8: "Seventh Day Adventist", + 9: "Church of England/Methodist", + 10: "Methodist/Church of England", + 11: "Church of England/Roman Catholic", + 12: "Church of England/United Reformed Church", + 13: "Roman Catholic/Church of England", + 14: "Quaker", + 15: "Christian", + 16: "United Reformed Church", + 17: "Congregational Church", + 18: "Free Church", + 19: "Church of England/Free Church", + 20: "Church of England/Christian", + 21: "Sikh", + 22: "Greek Orthodox", + 24: "Buddhist", + 25: "Hindu", + 26: "Moravian", + 28: "Inter- / non- denominational", + 29: "Multi-faith", + 30: "Church of England/Methodist/United Reform Church/Baptist", + 31: "Anglican", + 32: "Anglican/Christian", + 33: "Anglican/Evangelical", + 34: "Anglican/Church of England", + 35: "Catholic", + 36: "Charadi Jewish", + 37: "Christian/Evangelical", + 38: "Christian Science", + 39: "Christian/Methodist", + 40: "Christian/non-denominational", + 41: "Church of England/Evangelical", + 42: "Islam", + 43: "Orthodox Jewish", + 44: "Plymouth Brethren Christian Church", + 45: "Protestant", + 46: "Protestant/Evangelical", + 47: "Reformed Baptist", + 48: "Roman Catholic/Anglican", + 49: "Sunni Deobandi", +} + +ADMISSIONS_POLICY: dict[int, str] = { + 0: "Not applicable", + 2: "Selective", + 4: "Non-selective", +} + + +def translate(code, mapping: dict[int, str]) -> str | None: + """Translate a GIAS code to its display name. + + None/NaN -> None (column absent or suppressed). Unknown codes degrade to + "Unknown ()" with a warning so a new DfE value never blanks the UI. + """ + if code is None or (isinstance(code, float) and math.isnan(code)): + return None + code = int(code) + if code not in mapping: + logger.warning("Unknown GIAS code %s (not in dictionary)", code) + return f"Unknown ({code})" + return mapping[code] diff --git a/pipeline/transform/seeds/gias_code_names.csv b/pipeline/transform/seeds/gias_code_names.csv new file mode 100644 index 0000000..1b91038 --- /dev/null +++ b/pipeline/transform/seeds/gias_code_names.csv @@ -0,0 +1,105 @@ +field,code,name +school_type,1,Community school +school_type,2,Voluntary aided school +school_type,3,Voluntary controlled school +school_type,5,Foundation school +school_type,6,City technology college +school_type,7,Community special school +school_type,8,Non-maintained special school +school_type,10,Other independent special school +school_type,11,Other independent school +school_type,12,Foundation special school +school_type,14,Pupil referral unit +school_type,15,Local authority nursery school +school_type,18,Further education +school_type,24,Secure units +school_type,25,Offshore schools +school_type,26,Service children's education +school_type,27,Miscellaneous +school_type,28,Academy sponsor led +school_type,29,Higher education institutions +school_type,30,Welsh establishment +school_type,31,Sixth form centres +school_type,32,Special post 16 institution +school_type,33,Academy special sponsor led +school_type,34,Academy converter +school_type,35,Free schools +school_type,36,Free schools special +school_type,37,British schools overseas +school_type,38,Free schools alternative provision +school_type,39,Free schools 16 to 19 +school_type,40,University technical college +school_type,41,Studio schools +school_type,42,Academy alternative provision converter +school_type,43,Academy alternative provision sponsor led +school_type,44,Academy special converter +school_type,45,Academy 16-19 converter +school_type,46,Academy 16 to 19 sponsor led +school_type,49,Online provider +school_type,56,Institution funded by other government department +school_type,57,Academy secure 16 to 19 +establishment_status,1,Open +establishment_status,2,Closed +establishment_status,3,"Open, but proposed to close" +establishment_status,4,Proposed to open +phase_of_education,0,Not applicable +phase_of_education,1,Nursery +phase_of_education,2,Primary +phase_of_education,3,Middle deemed primary +phase_of_education,4,Secondary +phase_of_education,5,Middle deemed secondary +phase_of_education,6,16 plus +phase_of_education,7,All-through +official_sixth_form,0,Not applicable +official_sixth_form,1,Has a sixth form +official_sixth_form,2,Does not have a sixth form +religious_character,0,Does not apply +religious_character,2,Church of England +religious_character,3,Roman Catholic +religious_character,4,Methodist +religious_character,5,Jewish +religious_character,6,None +religious_character,7,Muslim +religious_character,8,Seventh Day Adventist +religious_character,9,Church of England/Methodist +religious_character,10,Methodist/Church of England +religious_character,11,Church of England/Roman Catholic +religious_character,12,Church of England/United Reformed Church +religious_character,13,Roman Catholic/Church of England +religious_character,14,Quaker +religious_character,15,Christian +religious_character,16,United Reformed Church +religious_character,17,Congregational Church +religious_character,18,Free Church +religious_character,19,Church of England/Free Church +religious_character,20,Church of England/Christian +religious_character,21,Sikh +religious_character,22,Greek Orthodox +religious_character,24,Buddhist +religious_character,25,Hindu +religious_character,26,Moravian +religious_character,28,Inter- / non- denominational +religious_character,29,Multi-faith +religious_character,30,Church of England/Methodist/United Reform Church/Baptist +religious_character,31,Anglican +religious_character,32,Anglican/Christian +religious_character,33,Anglican/Evangelical +religious_character,34,Anglican/Church of England +religious_character,35,Catholic +religious_character,36,Charadi Jewish +religious_character,37,Christian/Evangelical +religious_character,38,Christian Science +religious_character,39,Christian/Methodist +religious_character,40,Christian/non-denominational +religious_character,41,Church of England/Evangelical +religious_character,42,Islam +religious_character,43,Orthodox Jewish +religious_character,44,Plymouth Brethren Christian Church +religious_character,45,Protestant +religious_character,46,Protestant/Evangelical +religious_character,47,Reformed Baptist +religious_character,48,Roman Catholic/Anglican +religious_character,49,Sunni Deobandi +admissions_policy,0,Not applicable +admissions_policy,2,Selective +admissions_policy,4,Non-selective