diff --git a/backend/gias_codes.py b/backend/gias_codes.py index e454e8a..2216e1e 100644 --- a/backend/gias_codes.py +++ b/backend/gias_codes.py @@ -78,6 +78,7 @@ OFFICIAL_SIXTH_FORM: dict[int, str] = { 0: "Not applicable", 1: "Has a sixth form", 2: "Does not have a sixth form", + 9: "", } RELIGIOUS_CHARACTER: dict[int, str] = { @@ -128,12 +129,14 @@ RELIGIOUS_CHARACTER: dict[int, str] = { 47: "Reformed Baptist", 48: "Roman Catholic/Anglican", 49: "Sunni Deobandi", + 99: "", } ADMISSIONS_POLICY: dict[int, str] = { 0: "Not applicable", 2: "Selective", 4: "Non-selective", + 9: "", } diff --git a/backend/tests/test_gias_codes.py b/backend/tests/test_gias_codes.py index 95d02ed..e7305d0 100644 --- a/backend/tests/test_gias_codes.py +++ b/backend/tests/test_gias_codes.py @@ -81,3 +81,15 @@ def test_seed_matches_dictionaries(): for row in csv.DictReader(fh): seed[row["field"]][int(row["code"])] = row["name"] assert seed == fields + + +def test_blank_name_sentinel_codes_map_to_empty_string(): + """GIAS carries codes whose (name) column is blank — e.g. ReligiousCharacter + 99 (~4k schools) and AdmissionsPolicy 9 (~5.6k schools). The old name + pipeline served these as empty strings; the dictionaries must reproduce + that ("" is falsy, so UI tag heuristics stay silent) rather than letting + them hit the "Unknown ()" path meant for genuinely new codes.""" + assert RELIGIOUS_CHARACTER[99] == "" + assert ADMISSIONS_POLICY[9] == "" + assert translate(99, RELIGIOUS_CHARACTER) == "" + assert translate(9, ADMISSIONS_POLICY) == "" diff --git a/pipeline/scripts/generate_gias_codes.py b/pipeline/scripts/generate_gias_codes.py index bd1e135..80d0a26 100644 --- a/pipeline/scripts/generate_gias_codes.py +++ b/pipeline/scripts/generate_gias_codes.py @@ -94,13 +94,23 @@ def main() -> None: for code_col, name_col, dict_name, field_key in FIELDS: pairs = ( df[[code_col, name_col]] - .loc[lambda d: (d[code_col] != "") & (d[name_col] != "")] + .loc[lambda d: d[code_col] != ""] .drop_duplicates() ) - mapping = sorted((int(c), n) for c, n in pairs.itertuples(index=False)) - dupes = len(mapping) - len({c for c, _ in mapping}) - if dupes: - sys.exit(f"{code_col}: {dupes} codes map to multiple names — investigate before generating") + by_code: dict[int, set] = {} + for c, n in pairs.itertuples(index=False): + by_code.setdefault(int(c), set()).add(n) + mapping = [] + for code, names in sorted(by_code.items()): + named = sorted(n for n in names if n != "") + if len(named) > 1: + sys.exit(f"{code_col}: code {code} maps to multiple names {named} — investigate before generating") + # Codes that only ever appear with a blank (name) are GIAS + # "not recorded" sentinels (e.g. ReligiousCharacter 99, + # AdmissionsPolicy 9). Map them to "" so the API serves the same + # empty string the old name pipeline did — the "Unknown ()" + # path is reserved for genuinely new codes. + mapping.append((code, named[0] if named else "")) lines = [f"{dict_name}: dict[int, str] = {{"] for code, name in mapping: escaped = name.replace('"', '\\"') diff --git a/pipeline/scripts/gias_codes.py b/pipeline/scripts/gias_codes.py index e454e8a..2216e1e 100644 --- a/pipeline/scripts/gias_codes.py +++ b/pipeline/scripts/gias_codes.py @@ -78,6 +78,7 @@ OFFICIAL_SIXTH_FORM: dict[int, str] = { 0: "Not applicable", 1: "Has a sixth form", 2: "Does not have a sixth form", + 9: "", } RELIGIOUS_CHARACTER: dict[int, str] = { @@ -128,12 +129,14 @@ RELIGIOUS_CHARACTER: dict[int, str] = { 47: "Reformed Baptist", 48: "Roman Catholic/Anglican", 49: "Sunni Deobandi", + 99: "", } ADMISSIONS_POLICY: dict[int, str] = { 0: "Not applicable", 2: "Selective", 4: "Non-selective", + 9: "", } diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index d5dfd26..9a8832b 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -42,12 +42,12 @@ models: tests: - accepted_values: severity: warn - values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] + values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 99] - name: admissions_policy_code tests: - accepted_values: severity: warn - values: [0, 2, 4] + values: [0, 2, 4, 9] - name: dim_location description: School location dimension with PostGIS geometry diff --git a/pipeline/transform/seeds/gias_code_names.csv b/pipeline/transform/seeds/gias_code_names.csv index 1b91038..63809eb 100644 --- a/pipeline/transform/seeds/gias_code_names.csv +++ b/pipeline/transform/seeds/gias_code_names.csv @@ -53,6 +53,7 @@ phase_of_education,7,All-through official_sixth_form,0,Not applicable official_sixth_form,1,Has a sixth form official_sixth_form,2,Does not have a sixth form +official_sixth_form,9, religious_character,0,Does not apply religious_character,2,Church of England religious_character,3,Roman Catholic @@ -100,6 +101,8 @@ religious_character,46,Protestant/Evangelical religious_character,47,Reformed Baptist religious_character,48,Roman Catholic/Anglican religious_character,49,Sunni Deobandi +religious_character,99, admissions_policy,0,Not applicable admissions_policy,2,Selective admissions_policy,4,Non-selective +admissions_policy,9,