66 lines
2.4 KiB
Python
66 lines
2.4 KiB
Python
|
"""GIAS publishes its extracts in Windows-1252 and declares no charset.
|
|||
|
|
|
|||
|
|
The tap used to hand pandas `resp.text`, so requests guessed the codec.
|
|||
|
|
On 3 Oct 2026 it guessed windows-1250, and "St Thomas à Becket" was stored
|
|||
|
|
as "St Thomas ŕ Becket". The `encoding=` passed to read_csv did nothing,
|
|||
|
|
because the text was already decoded.
|
|||
|
|
"""
|
|||
|
|
import importlib.util
|
|||
|
|
import logging
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
|
|||
|
|
MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-gias'
|
|||
|
|
/ 'tap_uk_gias' / 'gias_csv.py')
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.fixture
|
|||
|
|
def gias_csv():
|
|||
|
|
spec = importlib.util.spec_from_file_location('gias_csv', MODULE)
|
|||
|
|
module = importlib.util.module_from_spec(spec)
|
|||
|
|
spec.loader.exec_module(module)
|
|||
|
|
return module
|
|||
|
|
|
|||
|
|
|
|||
|
|
# Byte for byte as GIAS writes it: 0xE0 à, 0x92 ’, 0xE9 é, 0xB0 °, 0xE7 ç.
|
|||
|
|
EXTRACT = (
|
|||
|
|
b'"URN","EstablishmentName","HeadLastName"\r\n'
|
|||
|
|
b'"138950","St Thomas \xe0 Becket Catholic Secondary School","Smith"\r\n'
|
|||
|
|
b'"100000","The Dean and Chapter of St Paul\x92s Cathedral","Pr\xe9vert"\r\n'
|
|||
|
|
b'"140677","North Star 180\xb0","Fran\xe7ois"\r\n'
|
|||
|
|
b'"100001","No head recorded",""\r\n'
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_names_decode_as_windows_1252(gias_csv):
|
|||
|
|
df = gias_csv.read_gias_csv(EXTRACT)
|
|||
|
|
assert list(df['EstablishmentName']) == [
|
|||
|
|
'St Thomas à Becket Catholic Secondary School',
|
|||
|
|
'The Dean and Chapter of St Paul’s Cathedral',
|
|||
|
|
'North Star 180°',
|
|||
|
|
'No head recorded',
|
|||
|
|
]
|
|||
|
|
assert list(df['HeadLastName']) == ['Smith', 'Prévert', 'François', '']
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_the_codec_requests_guessed_is_not_used(gias_csv):
|
|||
|
|
# What the tap stored on 3 Oct: the same bytes read as windows-1250.
|
|||
|
|
assert 'ŕ' in EXTRACT.decode('cp1250')
|
|||
|
|
names = ' '.join(gias_csv.read_gias_csv(EXTRACT)['EstablishmentName'])
|
|||
|
|
assert 'ŕ' not in names
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_values_stay_strings(gias_csv):
|
|||
|
|
df = gias_csv.read_gias_csv(EXTRACT)
|
|||
|
|
assert df.loc[0, 'URN'] == '138950'
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_a_byte_windows_1252_leaves_undefined_does_not_stop_the_load(gias_csv, caplog):
|
|||
|
|
# 0x81 has no Windows-1252 character. One odd name must not block the daily
|
|||
|
|
# refresh of every school, but it must be visible in the log.
|
|||
|
|
extract = b'"URN","EstablishmentName"\r\n"100002","Odd \x81 Name"\r\n'
|
|||
|
|
with caplog.at_level(logging.WARNING):
|
|||
|
|
df = gias_csv.read_gias_csv(extract, logger=logging.getLogger('gias'))
|
|||
|
|
assert df.loc[0, 'EstablishmentName'] == 'Odd � Name'
|
|||
|
|
assert 'could not be decoded' in caplog.text
|