166 lines
8.5 KiB
Python
166 lines
8.5 KiB
Python
"""Ofsted publishes further-education inspection outcomes on a gov.uk page per
|
||
academic year, separate from the school MI the tap already reads (audit H4).
|
||
The links below are the real ones from the 2025/26 page; the headers are
|
||
Ofsted's own.
|
||
"""
|
||
import importlib.util
|
||
from pathlib import Path
|
||
|
||
import pandas as pd
|
||
import pytest
|
||
|
||
MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-ofsted'
|
||
/ 'tap_uk_ofsted' / 'fe_mi.py')
|
||
|
||
A = 'https://assets.publishing.service.gov.uk/media/'
|
||
PAGE_2025_26 = f'''
|
||
<a href="{A}6989b16dc106cc5501b6e4c3/Management_information_-_further_education_and_skills_-_as_at_31_January_2026.csv">x</a>
|
||
<a href="{A}6989b18cd3f57710b50a9b92/Management_information_-_further_education_and_skills_-_10_November_2025_to_31_January_2026.csv">x</a>
|
||
<a href="{A}69aec8d6d620c14fa183eeb6/Management_information_-_further_education_and_skills_-__10_November_2025_to_28_February_2026.csv">x</a>
|
||
<a href="{A}69aec8fcc78869bf8eb8a54a/Management_information_-_further_education_and_skills_-_as_at_28_February_2026.csv">x</a>
|
||
<a href="{A}69aef091bde9c3f213c89a1e/Management_information_-_further_education_and_skills_-_as_at_31_January_2026.ods">x</a>
|
||
<a href="{A}6a9fec1e92e72b8ac437eef3/Management_information_-_further_education_and_skills_-_as_at_31_August_2026.csv">x</a>
|
||
<a href="{A}6a9fed6b5f6e942efe37eed4/Management_information_-_new_provider_monitoring_inspections_-_10_November_2025_to_31_August_2026.csv">x</a>
|
||
<a href="{A}6a9fed815f6e942efe37eed5/Management_information_-_monitoring_inspections_-_10_November_2025_to_31_August_2026.csv">x</a>
|
||
<a href="{A}6a9feda65f6e942efe37eed6/Management_information_-_further_education_and_skills_-_10_November_2025_to_31_August_2026.csv">x</a>
|
||
'''
|
||
|
||
SEARCH_RESULTS = [
|
||
{'title': 'Further education and skills inspections and outcomes: management information from September 2024 to August 2025',
|
||
'link': '/government/statistical-data-sets/fe-2024-25'},
|
||
{'title': 'Further education and skills inspections and outcomes: management information from November 2025 to August 2026',
|
||
'link': '/government/statistical-data-sets/fe-2025-26'},
|
||
{'title': 'Further education and skills inspections and outcomes: management information from September 2023 to August 2024',
|
||
'link': '/government/statistical-data-sets/fe-2023-24'},
|
||
{'title': 'Monthly management information: Ofsted\'s school inspections outcomes',
|
||
'link': '/government/statistical-data-sets/schools'},
|
||
]
|
||
|
||
PROVIDER_HEADER = (
|
||
'Provider URN', 'Provider UKPRN', 'Provider name', 'Provider type', 'Provider group',
|
||
'Local authority', 'Region', 'Ofsted region',
|
||
'Has the provider received a full inspection under the renewed EIF?',
|
||
'Has the provider received a new provider monitoring inspection?',
|
||
'Date of latest short inspection before renewed EIF (if relevant)',
|
||
'Inspection number of latest full inspection before renewed EIF',
|
||
'First day of inspection of latest inspection before renewed EIF',
|
||
'Latest overall effectiveness before renewed EIF',
|
||
'Latest quality of education before renewed EIF',
|
||
'Latest behaviour and attitudes before renewed EIF',
|
||
'Latest personal development before renewed EIF',
|
||
'Latest leadership and management before renewed EIF',
|
||
'Latest contribution to meeting skills needs before renewed EIF',
|
||
'Latest safeguarding before renewed EIF', '',
|
||
)
|
||
|
||
INSPECTION_HEADER = (
|
||
'Provider URN', 'Provider UKPRN', 'Provider name', 'Provider type', 'Provider group',
|
||
'Local authority', 'Region', 'Ofsted region', 'Inspection number', 'First day of inspection',
|
||
'Date published', 'Safeguarding standards', 'Inclusion', 'Leadership and governance',
|
||
'Contribution to meeting skills needs',
|
||
'Education programmes for young people - curriculum, teaching and training',
|
||
'Education programmes for young people - achievement',
|
||
'Education programmes for young people - participation and development',
|
||
'Adult learning programmes - curriculum, teaching and training',
|
||
'Adult learning programmes - achievement',
|
||
'Adult learning programmes - participation and development',
|
||
'Apprenticeships - curriculum, teaching and training', 'Apprenticeships - achievement',
|
||
'Apprenticeships - participation and development',
|
||
'Provision for learners and apprentices with high needs - curriculum, teaching and training',
|
||
'Provision for learners and apprentices with high needs - achievement',
|
||
'Provision for learners and apprentices with high needs - participation and development', '',
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def fe():
|
||
spec = importlib.util.spec_from_file_location('fe_mi', MODULE)
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
def test_the_newest_as_at_file_is_chosen(fe):
|
||
assert fe.latest_as_at_csv(PAGE_2025_26).endswith('_as_at_31_August_2026.csv')
|
||
|
||
|
||
def test_the_latest_cumulative_inspections_file_is_chosen_not_a_monitoring_one(fe):
|
||
assert fe.latest_period_csv(PAGE_2025_26) == (
|
||
A + '6a9feda65f6e942efe37eed6/Management_information_-_further_education_and_skills_'
|
||
'-_10_November_2025_to_31_August_2026.csv')
|
||
|
||
|
||
def test_a_doubled_underscore_file_name_still_parses(fe):
|
||
page = '\n'.join(line for line in PAGE_2025_26.splitlines() if 'August' not in line)
|
||
assert fe.latest_period_csv(page).endswith('__10_November_2025_to_28_February_2026.csv')
|
||
|
||
|
||
def test_monitoring_files_alone_give_nothing(fe):
|
||
page = '\n'.join(line for line in PAGE_2025_26.splitlines() if 'monitoring' in line)
|
||
assert fe.latest_period_csv(page) is None
|
||
assert fe.latest_as_at_csv(page) is None
|
||
|
||
|
||
def test_pages_are_the_fe_series_newest_first(fe):
|
||
assert fe.fe_pages(SEARCH_RESULTS) == [
|
||
'/government/statistical-data-sets/fe-2025-26',
|
||
'/government/statistical-data-sets/fe-2024-25',
|
||
'/government/statistical-data-sets/fe-2023-24',
|
||
]
|
||
|
||
|
||
def test_report_cards_are_read_only_from_renewed_framework_pages(fe):
|
||
# Earlier pages hold previous-framework inspection lists, not report cards.
|
||
assert fe.renewed_framework_pages(SEARCH_RESULTS) == ['/government/statistical-data-sets/fe-2025-26']
|
||
|
||
|
||
def test_providers_fall_back_to_the_previous_page(fe):
|
||
pages = {'/new': '<p>No files yet</p>', '/old': PAGE_2025_26}
|
||
assert fe.providers_csv_url(['/new', '/old'], pages.__getitem__).endswith('_as_at_31_August_2026.csv')
|
||
|
||
|
||
@pytest.mark.parametrize('columns, header', [('PROVIDER_COLUMNS', PROVIDER_HEADER),
|
||
('INSPECTION_COLUMNS', INSPECTION_HEADER)])
|
||
def test_every_mapped_column_is_in_ofsteds_file(fe, columns, header):
|
||
assert set(getattr(fe, columns)) - set(header) == set()
|
||
|
||
|
||
def test_csv_reading_and_records(fe):
|
||
content = ('' + ','.join(f'"{c}"' for c in PROVIDER_HEADER[:14]) + '\n'
|
||
+ '145175,10005429,Queen Mary\'s College,Sixth form college,Sixth form colleges,Hampshire,'
|
||
'South East,South East,No,No,-,10227183,06/12/2022,2\n'
|
||
+ ',,Blank provider,,,,,,,,,,,\n').encode('utf-8')
|
||
rows = list(fe.records(fe.read_mi_csv(content), fe.PROVIDER_COLUMNS, 'provider_urn'))
|
||
assert rows == [{
|
||
'provider_urn': '145175', 'provider_ukprn': '10005429', 'provider_name': "Queen Mary's College",
|
||
'provider_type': 'Sixth form college', 'renewed_eif_full_inspection': 'No',
|
||
'graded_inspection_date': '06/12/2022', 'overall_effectiveness': '2', 'short_inspection_date': '-',
|
||
}]
|
||
|
||
|
||
def test_a_gov_uk_outage_yields_no_providers_rather_than_failing(fe):
|
||
# The FE streams share the monthly Ofsted extract with the school MI; an
|
||
# error here must not fail that extract (review of H4).
|
||
def down():
|
||
raise ConnectionError('gov.uk unreachable')
|
||
logged = []
|
||
assert list(fe.provider_records(down, lambda page: '', lambda url: b'', logged.append)) == []
|
||
assert logged
|
||
|
||
|
||
def test_one_unreachable_page_does_not_stop_the_other_report_cards(fe):
|
||
results = [{'title': fe.PAGE_TITLE_PREFIX + 'September 2026 to August 2027', 'link': '/fe-2026-27'},
|
||
{'title': fe.PAGE_TITLE_PREFIX + 'November 2025 to August 2026', 'link': '/fe-2025-26'}]
|
||
csv = (','.join(INSPECTION_HEADER[:12]) + '\n'
|
||
+ '143526,10028480,The Autism Project,,,,,,10427540,12/05/2026,22/07/2026,Met\n').encode()
|
||
|
||
def page(path):
|
||
if path == '/fe-2026-27':
|
||
raise ConnectionError('timeout')
|
||
return PAGE_2025_26
|
||
|
||
logged = []
|
||
rows = list(fe.inspection_records(lambda: results, page, lambda url: csv, logged.append))
|
||
assert [r['inspection_number'] for r in rows] == ['10427540']
|
||
assert logged
|