"""GIAS publishes its extracts in Windows-1252 and declares no charset. The tap used to hand pandas `resp.text`, so requests guessed the codec. On 3 Oct 2026 it guessed windows-1250, and "St Thomas à Becket" was stored as "St Thomas ŕ Becket". The `encoding=` passed to read_csv did nothing, because the text was already decoded. """ import importlib.util import logging from pathlib import Path import pytest MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-gias' / 'tap_uk_gias' / 'gias_csv.py') @pytest.fixture def gias_csv(): spec = importlib.util.spec_from_file_location('gias_csv', MODULE) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module # Byte for byte as GIAS writes it: 0xE0 à, 0x92 ’, 0xE9 é, 0xB0 °, 0xE7 ç. EXTRACT = ( b'"URN","EstablishmentName","HeadLastName"\r\n' b'"138950","St Thomas \xe0 Becket Catholic Secondary School","Smith"\r\n' b'"100000","The Dean and Chapter of St Paul\x92s Cathedral","Pr\xe9vert"\r\n' b'"140677","North Star 180\xb0","Fran\xe7ois"\r\n' b'"100001","No head recorded",""\r\n' ) def test_names_decode_as_windows_1252(gias_csv): df = gias_csv.read_gias_csv(EXTRACT) assert list(df['EstablishmentName']) == [ 'St Thomas à Becket Catholic Secondary School', 'The Dean and Chapter of St Paul’s Cathedral', 'North Star 180°', 'No head recorded', ] assert list(df['HeadLastName']) == ['Smith', 'Prévert', 'François', ''] def test_the_codec_requests_guessed_is_not_used(gias_csv): # What the tap stored on 3 Oct: the same bytes read as windows-1250. assert 'ŕ' in EXTRACT.decode('cp1250') names = ' '.join(gias_csv.read_gias_csv(EXTRACT)['EstablishmentName']) assert 'ŕ' not in names def test_values_stay_strings(gias_csv): df = gias_csv.read_gias_csv(EXTRACT) assert df.loc[0, 'URN'] == '138950' def test_a_byte_windows_1252_leaves_undefined_does_not_stop_the_load(gias_csv, caplog): # 0x81 has no Windows-1252 character. One odd name must not block the daily # refresh of every school, but it must be visible in the log. extract = b'"URN","EstablishmentName"\r\n"100002","Odd \x81 Name"\r\n' with caplog.at_level(logging.WARNING): df = gias_csv.read_gias_csv(extract, logger=logging.getLogger('gias')) assert df.loc[0, 'EstablishmentName'] == 'Odd � Name' assert 'could not be decoded' in caplog.text