diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/fe_mi.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/fe_mi.py index 9ba7df2..c9705b9 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/fe_mi.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/fe_mi.py @@ -135,3 +135,54 @@ def records(df: pd.DataFrame, columns: dict[str, str], key: str) -> Iterator[dic record = {field: str(row[source]).strip() for source, field in present.items()} if record.get(key): yield record + + +# requests' exceptions are OSErrors; a body that is not JSON is a ValueError. +_UNAVAILABLE = (OSError, ValueError) + + +def provider_records( + search: Callable[[], list[dict]], + fetch_page: Callable[[str], str], + fetch_csv: Callable[[str], bytes], + log: Callable[[str], None], +) -> Iterator[dict]: + """The newest "as at" file's provider records. gov.uk being unreachable + yields nothing, logged: the FE streams share the monthly Ofsted extract + with the school MI and must not fail it.""" + try: + url = providers_csv_url(fe_pages(search()), fetch_page) + if not url: + log("No Ofsted FE 'as at' file found") + return + df = read_mi_csv(fetch_csv(url)) + except _UNAVAILABLE as exc: + log(f"Ofsted FE providers MI unavailable: {exc}") + return + yield from records(df, PROVIDER_COLUMNS, "provider_urn") + + +def inspection_records( + search: Callable[[], list[dict]], + fetch_page: Callable[[str], str], + fetch_csv: Callable[[str], bytes], + log: Callable[[str], None], +) -> Iterator[dict]: + """Renewed-framework inspection records from every page since November + 2025. A page that cannot be read is logged and skipped; the others load.""" + try: + pages = renewed_framework_pages(search()) + except _UNAVAILABLE as exc: + log(f"Ofsted FE inspections MI unavailable: {exc}") + return + for page in pages: + try: + url = latest_period_csv(fetch_page(page)) + if not url: + log(f"No renewed-framework FE inspections file on {page}") + continue + df = read_mi_csv(fetch_csv(url)) + except _UNAVAILABLE as exc: + log(f"Ofsted FE inspections MI unavailable for {page}: {exc}") + continue + yield from records(df, INSPECTION_COLUMNS, "inspection_number") diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index ae91d29..6df1f5b 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -15,12 +15,8 @@ from tap_uk_ofsted.fe_mi import ( PROVIDER_COLUMNS, SEARCH_QUERY, SEARCH_URL, - fe_pages, - latest_period_csv, - providers_csv_url, - read_mi_csv, - records, - renewed_framework_pages, + inspection_records, + provider_records, ) GOV_UK_PAGE = ( @@ -331,6 +327,12 @@ def _gov_uk_page(path: str) -> str: return resp.text +def _gov_uk_file(url: str) -> bytes: + resp = requests.get(url, timeout=120) + resp.raise_for_status() + return resp.content + + class OfstedFEProvidersStream(Stream): """Ofsted's further education and skills providers: the newest "as at" file, one row per provider with its latest inspection under the previous @@ -346,14 +348,7 @@ class OfstedFEProvidersStream(Stream): ).to_dict() def get_records(self, context): - url = providers_csv_url(fe_pages(_fe_search_results()), _gov_uk_page) - if not url: - self.logger.error("No Ofsted FE 'as at' file found") - return - self.logger.info("Downloading Ofsted FE providers MI: %s", url) - resp = requests.get(url, timeout=120) - resp.raise_for_status() - yield from records(read_mi_csv(resp.content), PROVIDER_COLUMNS, "provider_urn") + yield from provider_records(_fe_search_results, _gov_uk_page, _gov_uk_file, self.logger.error) class OfstedFEInspectionsStream(Stream): @@ -371,15 +366,7 @@ class OfstedFEInspectionsStream(Stream): ).to_dict() def get_records(self, context): - for page in renewed_framework_pages(_fe_search_results()): - url = latest_period_csv(_gov_uk_page(page)) - if not url: - self.logger.warning("No renewed-framework FE inspections file on %s", page) - continue - self.logger.info("Downloading Ofsted FE inspections MI: %s", url) - resp = requests.get(url, timeout=120) - resp.raise_for_status() - yield from records(read_mi_csv(resp.content), INSPECTION_COLUMNS, "inspection_number") + yield from inspection_records(_fe_search_results, _gov_uk_page, _gov_uk_file, self.logger.warning) class TapUKOfsted(Tap): diff --git a/pipeline/tests/test_ofsted_fe_mi.py b/pipeline/tests/test_ofsted_fe_mi.py index 7fd9310..c182a2f 100644 --- a/pipeline/tests/test_ofsted_fe_mi.py +++ b/pipeline/tests/test_ofsted_fe_mi.py @@ -136,3 +136,30 @@ def test_csv_reading_and_records(fe): 'provider_type': 'Sixth form college', 'renewed_eif_full_inspection': 'No', 'graded_inspection_date': '06/12/2022', 'overall_effectiveness': '2', 'short_inspection_date': '-', }] + + +def test_a_gov_uk_outage_yields_no_providers_rather_than_failing(fe): + # The FE streams share the monthly Ofsted extract with the school MI; an + # error here must not fail that extract (review of H4). + def down(): + raise ConnectionError('gov.uk unreachable') + logged = [] + assert list(fe.provider_records(down, lambda page: '', lambda url: b'', logged.append)) == [] + assert logged + + +def test_one_unreachable_page_does_not_stop_the_other_report_cards(fe): + results = [{'title': fe.PAGE_TITLE_PREFIX + 'September 2026 to August 2027', 'link': '/fe-2026-27'}, + {'title': fe.PAGE_TITLE_PREFIX + 'November 2025 to August 2026', 'link': '/fe-2025-26'}] + csv = (','.join(INSPECTION_HEADER[:12]) + '\n' + + '143526,10028480,The Autism Project,,,,,,10427540,12/05/2026,22/07/2026,Met\n').encode() + + def page(path): + if path == '/fe-2026-27': + raise ConnectionError('timeout') + return PAGE_2025_26 + + logged = [] + rows = list(fe.inspection_records(lambda: results, page, lambda url: csv, logged.append)) + assert [r['inspection_number'] for r in rows] == ['10427540'] + assert logged