Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0269eec9c4 | ||
|
|
81de008e4a | ||
|
|
9feb37b974 | ||
|
|
ab25d83b9d | ||
|
|
69da9d84db |
No files matched your search
@@ -1048,6 +1048,11 @@ async def get_school_details(request: Request, urn: int):
|
||||
"age_range": latest.get("age_range", ""),
|
||||
"has_sixth_form": latest.get("has_sixth_form"),
|
||||
"nursery_provision": latest.get("nursery_provision"),
|
||||
# What the Ofsted badge may say when there is no inspection record
|
||||
# (audit H4): the inspectorate GIAS names, and whether Ofsted's own
|
||||
# data lists the school at all.
|
||||
"inspectorate": latest.get("inspectorate"),
|
||||
"ofsted_listed": latest.get("ofsted_listed"),
|
||||
# The header's Selective flag reads it (lib/schoolFacts).
|
||||
"admissions_policy": latest.get("admissions_policy"),
|
||||
"status": latest.get("status"),
|
||||
@@ -1210,6 +1215,9 @@ async def compare_schools(
|
||||
# schools get entrance-test framing, never the distance template.
|
||||
"admissions_policy": convert_to_native(latest.get("admissions_policy")),
|
||||
"has_sixth_form": convert_to_native(latest.get("has_sixth_form")),
|
||||
# Audit H4: what compare may say when there is no inspection record.
|
||||
"inspectorate": convert_to_native(latest.get("inspectorate")),
|
||||
"ofsted_listed": convert_to_native(latest.get("ofsted_listed")),
|
||||
"capacity": convert_to_native(latest.get("capacity")),
|
||||
"gias_total_pupils": convert_to_native(latest.get("gias_total_pupils")),
|
||||
"trust_name": convert_to_native(latest.get("trust_name")),
|
||||
|
||||
@@ -251,6 +251,8 @@ _MAIN_QUERY = text("""
|
||||
s.website,
|
||||
s.telephone,
|
||||
s.nursery_provision,
|
||||
s.inspectorate,
|
||||
s.ofsted_listed,
|
||||
foi.current_grade AS ofsted_grade,
|
||||
foi.current_grade_date AS ofsted_grade_date,
|
||||
foi.latest_visit_date AS ofsted_date,
|
||||
@@ -376,7 +378,8 @@ for _old, _new in _LEGACY_NAME_REPLACEMENTS:
|
||||
_MAIN_QUERY_LEGACY_NAMES = text(_MAIN_QUERY_LEGACY_NAMES)
|
||||
|
||||
# Fallback used when marts predate the optional GIAS/location detail columns
|
||||
# (telephone, nursery provision, county, parliamentary constituency) — i.e. the
|
||||
# (telephone, nursery provision, county, parliamentary constituency, and the
|
||||
# inspectorate / Ofsted-listing columns added for audit H4) — i.e. the
|
||||
# nightly dbt pipeline hasn't rebuilt the mart yet on this DB. Keeps each column
|
||||
# present as NULL so the whole data load degrades gracefully instead of failing.
|
||||
_EXTRA_OPTIONAL_COLUMNS = (
|
||||
@@ -384,6 +387,8 @@ _EXTRA_OPTIONAL_COLUMNS = (
|
||||
"nursery_provision",
|
||||
"county",
|
||||
"parliamentary_constituency",
|
||||
"inspectorate",
|
||||
"ofsted_listed",
|
||||
)
|
||||
_MAIN_QUERY_NO_EXTRA_COLS = str(_MAIN_QUERY)
|
||||
for _col, _tbl in (
|
||||
@@ -391,6 +396,8 @@ for _col, _tbl in (
|
||||
("nursery_provision", "s"),
|
||||
("county", "l"),
|
||||
("parliamentary_constituency", "l"),
|
||||
("inspectorate", "s"),
|
||||
("ofsted_listed", "s"),
|
||||
):
|
||||
_src = f"{_tbl}.{_col},"
|
||||
assert _src in _MAIN_QUERY_NO_EXTRA_COLS, (
|
||||
|
||||
@@ -39,6 +39,9 @@ def report_card_labels(ofsted: dict) -> dict:
|
||||
|
||||
|
||||
def ofsted_page_url(urn: int) -> str:
|
||||
"""The school's page on ofsted.gov.uk (all its reports live there —
|
||||
we never deep-link an individual report)."""
|
||||
return f"https://reports.ofsted.gov.uk/provider/21/{urn}"
|
||||
"""The school's page on ofsted.gov.uk (all its reports live there — we
|
||||
never deep-link an individual report). Ofsted's provider pages sit under a
|
||||
category (21 primary, 23 secondary, 25 special, 40 FE colleges, …); this
|
||||
address redirects to the right one, where provider/21 404s for every
|
||||
non-primary school."""
|
||||
return f"https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/{urn}"
|
||||
@@ -578,6 +578,8 @@ SCHOOL_COLUMNS = [
|
||||
"ofsted_date",
|
||||
"ofsted_framework",
|
||||
"ofsted_rc_date",
|
||||
"inspectorate",
|
||||
"ofsted_listed",
|
||||
"latitude",
|
||||
"longitude",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Audit H4: every payload that feeds an Ofsted badge carries the inspectorate
|
||||
GIAS names and whether Ofsted's own data lists the school, so the site never
|
||||
says "Not yet inspected" on the strength of missing data."""
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def _df():
|
||||
base = dict(year=202425, local_authority='Cambridgeshire', local_authority_code=873,
|
||||
postcode='CB1 1AA', latitude=52.2, longitude=0.12, address='1 High St',
|
||||
status='Open', town='Cambridge')
|
||||
return pd.DataFrame([
|
||||
dict(base, urn=136083, school_name='Abbey College Cambridge', phase='Secondary',
|
||||
school_type='Other independent school', inspectorate='ISI', ofsted_listed=False),
|
||||
dict(base, urn=150031, school_name='Abbas and Templecombe Church School', phase='Primary',
|
||||
school_type='Academy converter', inspectorate='Not applicable', ofsted_listed=True),
|
||||
])
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(monkeypatch):
|
||||
from backend import app as app_module
|
||||
from backend import database as database_module
|
||||
|
||||
def _no_db():
|
||||
raise RuntimeError('no database in this test')
|
||||
|
||||
monkeypatch.setattr(app_module, 'load_latest_school_data', _df)
|
||||
monkeypatch.setattr(app_module, 'load_school_data', _df)
|
||||
monkeypatch.setattr(app_module, 'get_supplementary_data', lambda db, urn: {})
|
||||
monkeypatch.setattr(app_module, '_place_registry', None)
|
||||
monkeypatch.setattr(database_module, 'SessionLocal', _no_db)
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def test_the_list_carries_inspectorate_and_listing(client):
|
||||
resp = client.get('/api/schools?page_size=50')
|
||||
assert resp.status_code == 200, resp.text
|
||||
by_urn = {s['urn']: s for s in resp.json()['schools']}
|
||||
assert by_urn[136083]['inspectorate'] == 'ISI'
|
||||
assert by_urn[150031]['ofsted_listed'] is True
|
||||
|
||||
|
||||
def test_the_school_page_carries_them(client):
|
||||
info = client.get('/api/schools/136083').json()['school_info']
|
||||
assert info['inspectorate'] == 'ISI'
|
||||
assert info['ofsted_listed'] is False
|
||||
|
||||
|
||||
def test_compare_carries_them(client):
|
||||
body = client.get('/api/compare?urns=136083,150031').json()
|
||||
assert body['comparison']['150031']['school_info']['ofsted_listed'] is True
|
||||
assert body['comparison']['136083']['school_info']['inspectorate'] == 'ISI'
|
||||
|
||||
|
||||
def test_a_mart_built_before_the_columns_still_loads():
|
||||
# Missing columns make the main query retry with NULLs (data_loader).
|
||||
from backend import data_loader
|
||||
assert {'inspectorate', 'ofsted_listed'} <= set(data_loader._EXTRA_OPTIONAL_COLUMNS)
|
||||
fallback = str(data_loader._MAIN_QUERY_NO_EXTRA_COLS)
|
||||
assert 'NULL AS inspectorate,' in fallback and 'NULL AS ofsted_listed,' in fallback
|
||||
@@ -42,4 +42,4 @@ def test_unknown_code_is_skipped_not_crashed():
|
||||
|
||||
|
||||
def test_provider_url():
|
||||
assert ofsted_page_url(138690) == "https://reports.ofsted.gov.uk/provider/21/138690"
|
||||
assert ofsted_page_url(138690) == "https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/138690"
|
||||
@@ -33,7 +33,7 @@ def test_report_card_block_and_provider_url():
|
||||
assert block["report_card"]["rc_achievement"]["label"] == "Strong standard"
|
||||
assert "rc_safeguarding_met" not in block["report_card"]
|
||||
assert block["rc_safeguarding_met"] is True
|
||||
assert block["ofsted_page_url"] == "https://reports.ofsted.gov.uk/provider/21/100140"
|
||||
assert block["ofsted_page_url"] == "https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/100140"
|
||||
|
||||
|
||||
def test_no_grade_is_carried_past_a_newer_inspection():
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,256 +0,0 @@
|
||||
# Inspection Status Beyond Ofsted's School Data — Design
|
||||
|
||||
**Date:** 2026-10-06
|
||||
**Status:** approved design, not yet implemented
|
||||
**Scope:** `tap_uk_gias` (two fields), `tap_uk_ofsted` (two FE streams), Ofsted
|
||||
and GIAS staging, `int_ofsted_latest` / `fact_ofsted_latest`, new
|
||||
`int_ofsted_fe_latest` and `int_ofsted_listed`, `dim_school`, monthly Ofsted DAG
|
||||
selector, backend list and detail payloads, search badge, map card, school page
|
||||
Ofsted section, compare Ofsted rows
|
||||
**Fixes:** audit finding H4, from the 3 Oct 2026 accuracy audit
|
||||
|
||||
## Goal
|
||||
|
||||
Never label a school "Not yet inspected" unless Ofsted's own data lists it
|
||||
without an inspection, and show every inspection result we can source, with its
|
||||
date.
|
||||
|
||||
## The problem
|
||||
|
||||
The search badge and the map card (`buildOfstedListBadge`,
|
||||
`nextjs-app/lib/utils.ts`) print "Not yet inspected" whenever we hold no Ofsted
|
||||
record. The school page hides its Ofsted section for these schools, and compare
|
||||
says "No inspection in our dataset". On production on 6 October 2026 (build
|
||||
`2b5e681`), 2,587 listed schools have no Ofsted record. Joined with GIAS
|
||||
(6 October 2026) and Ofsted's management information (31 August 2026):
|
||||
|
||||
| Group | Schools | Evidence |
|
||||
|---|---|---|
|
||||
| Independent schools inspected by ISI | 1,341 | GIAS `InspectorateName` = "ISI". GIAS holds no date, outcome or report link for any of them; ISI publishes no bulk data. |
|
||||
| Further-education and post-16 providers inspected by Ofsted | 519 | In Ofsted's "Further education and skills" MI: 509 by Provider URN, 10 only by UKPRN through GIAS. 207 FE colleges, 129 special post-16 institutions, 97 higher-education institutions (their FE and apprenticeship provision), 78 16–19 academies and free schools, 8 others. Queen Mary's College (145175): Good, 6 December 2022. |
|
||||
| Listed by Ofsted, never inspected | 263+ | New academies and free schools present in Ofsted's school MI with every inspection field `NULL`, e.g. Abbas and Templecombe Church School (150031, opened January 2024). "Not yet inspected" is Ofsted's own position for these. |
|
||||
| No usable inspection data | about 460 | Secure units (inspected as children's homes), higher-education institutions outside the FE MI, newer independents, schools opened after the MI date. |
|
||||
|
||||
Of the 519 FE providers: 443 have only a grade from the previous framework
|
||||
(overall effectiveness 1–4 and the inspection's first day); 45 also have a newer
|
||||
report card under the renewed framework (from 10 November 2025); 2 have only a
|
||||
report card; 29 are listed with no inspection. 116 FE providers nationally have
|
||||
a short inspection after their latest full inspection; the MI gives its date
|
||||
but not its outcome.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Area grades from the previous FE framework, and the report card's
|
||||
programme-by-theme grid. The FE display is the headline and a link.
|
||||
- ISI dates or outcomes (no data source).
|
||||
- A per-school ISI link: ISI's pages use ISI's own identifiers, not URNs.
|
||||
- Inspections of secure units and children's homes (social care framework).
|
||||
- Changing how school-framework records are chosen (the C1 rule stands).
|
||||
|
||||
## Data
|
||||
|
||||
### Extractors
|
||||
|
||||
**GIAS (`tap_uk_gias`).** Declare `UKPRN` and `InspectorateName (name)` on the
|
||||
establishments stream. They are plain values, not codes, so the generated GIAS
|
||||
dictionaries (backend and pipeline copies, parity-tested) are untouched.
|
||||
|
||||
**Ofsted (`tap_uk_ofsted`).** Two streams from Ofsted's "Further education and
|
||||
skills inspections and outcomes: management information from …" pages on
|
||||
gov.uk. Pages are found through the gov.uk search API by that title prefix; a
|
||||
new page appears each academic year.
|
||||
|
||||
- `ofsted_fe_providers`: the newest page's newest
|
||||
`…further_education_and_skills_-_as_at_<date>.csv`. One row per provider:
|
||||
`Provider URN`, `Provider UKPRN`, `Provider name`, `Provider type`,
|
||||
`Has the provider received a full inspection under the renewed EIF?`,
|
||||
`First day of inspection of latest inspection before renewed EIF`,
|
||||
`Latest overall effectiveness before renewed EIF`,
|
||||
`Date of latest short inspection before renewed EIF (if relevant)`.
|
||||
Key: `provider_urn`.
|
||||
- `ofsted_fe_inspections`: renewed-framework inspections. Each page carries
|
||||
cumulative files `…further_education_and_skills_-_<start>_to_<end>.csv`
|
||||
(`10_November_2025_to_31_January_2026`, `…_to_28_February_2026`, …, sometimes
|
||||
with a doubled underscore); the stream reads the file with the latest end date
|
||||
from every page since November 2025, so a report card survives the next
|
||||
year's page. Files named `new_provider_monitoring` or `monitoring_inspections`
|
||||
are excluded. Fields: `Provider URN`, `Provider UKPRN`, `Inspection number`,
|
||||
`First day of inspection`, `Date published`, `Safeguarding standards`.
|
||||
Key: `inspection_number`.
|
||||
|
||||
Page and file selection, and the CSV-to-field mapping, live in a module that
|
||||
imports only pandas and the standard library, so CI's pytest (no
|
||||
`singer_sdk`) can test them against saved real page HTML and real headers.
|
||||
|
||||
The existing school-MI stream is unchanged. It already loads every row Ofsted
|
||||
lists, inspected or not; `stg_ofsted_inspections` drops the uninspected rows.
|
||||
|
||||
### dbt
|
||||
|
||||
- `stg_gias_establishments` gains `ukprn` and `inspectorate_name`.
|
||||
- `stg_ofsted_fe_providers` and `stg_ofsted_fe_inspections`: dates parsed as
|
||||
`DD/MM/YYYY`; grades `1`–`4` cast to integers and `-` to null; safeguarding
|
||||
`Met` / `Not met` to a boolean.
|
||||
- `int_ofsted_fe_latest`, one row per URN we hold. A provider matches a school
|
||||
by `Provider URN` = URN; failing that, by `Provider UKPRN` = the school's GIAS
|
||||
`ukprn`. The rule, in the terms of the C1 current-status rule:
|
||||
|
||||
| FE record | `current_grade` | `latest_visit` |
|
||||
|---|---|---|
|
||||
| Report card newer than the previous-framework inspection (or no such inspection) | null; `rc_inspection_date`, `rc_safeguarding_met` set | the report card (`report_card`) |
|
||||
| Previous-framework grade 1–4, no newer report card | that grade, dated by the inspection's first day, basis `graded` | the later of that inspection (`graded`) and any short inspection (`short`) |
|
||||
| Grade `-` and no report card | — | no row |
|
||||
|
||||
A short inspection's outcome is not in the data, so it is a visit, never a
|
||||
grade change.
|
||||
- `int_ofsted_latest` / `fact_ofsted_latest` take FE rows only for URNs with no
|
||||
school-framework record; a school record always wins. FE rows carry
|
||||
`framework = 'Further education and skills'`; school area and report-card
|
||||
area columns stay null. `latest_visit_kind` gains the value `short`.
|
||||
- `int_ofsted_listed`: every URN Ofsted's data includes, inspected or not —
|
||||
all rows of `raw.ofsted_inspections` (before staging's date filter) plus every
|
||||
matched FE provider.
|
||||
- `dim_school` gains `inspectorate` (GIAS `inspectorate_name`: "ISI", "Ofsted",
|
||||
"Not applicable" or null) and `ofsted_listed` (boolean), the latter joined by
|
||||
the same optional-relation pattern used for `int_ofsted_latest`.
|
||||
`pipeline/tests/test_dag_selectors.py`'s `OPTIONAL_PARENTS` adds
|
||||
`int_ofsted_listed`.
|
||||
- The monthly Ofsted DAG build selects
|
||||
`stg_ofsted_fe_providers+ stg_ofsted_fe_inspections+ int_ofsted_listed+` as
|
||||
well.
|
||||
- Tests: the existing `fact_ofsted_latest` tests (unique URN,
|
||||
`assert_ofsted_current_grade_consistent`) cover FE rows. A new test asserts no
|
||||
URN draws on both a school and an FE record.
|
||||
|
||||
## What each page shows
|
||||
|
||||
**Search badge and map card.** `buildOfstedListBadge` may return no badge. In
|
||||
order:
|
||||
|
||||
1. report card → "Report Card · year";
|
||||
2. grade in force → grade word · year;
|
||||
3. inspected, no grade in force → "Inspected · year";
|
||||
4. GIAS inspectorate "ISI" → "Inspected by ISI" (grey);
|
||||
5. listed by Ofsted → "Not yet inspected" (grey);
|
||||
6. otherwise no badge.
|
||||
|
||||
FE schools reach 1–3 with the same labels: Queen Mary's College "Good · 2022";
|
||||
The Autism Project – CareTrade (143526) "Report Card · 2026".
|
||||
|
||||
**School page.**
|
||||
|
||||
- FE: the existing Ofsted section with "Further education and skills" under the
|
||||
heading. A grade reads as today ("Good", "Graded inspection, 6 December
|
||||
2022"); a report card shows its date and the safeguarding result; a later
|
||||
short inspection gets "Latest visit: Short inspection, [date]". No area or
|
||||
programme grid. The existing "Ofsted reports ↗" link resolves for FE
|
||||
providers (Leyton Sixth Form College 130457 → `/provider/40/130457`).
|
||||
- ISI: an inspection section reading "Inspected by the Independent Schools
|
||||
Inspectorate (ISI)" with "ISI reports ↗" linking to
|
||||
`https://www.isi.net/reports/`.
|
||||
- Listed, never inspected: an Ofsted section reading "Not yet inspected", with
|
||||
the Ofsted reports link (for a new academy, Ofsted's page links the
|
||||
predecessor school's reports).
|
||||
- No data: no section, as today.
|
||||
|
||||
**Compare.** FE and listed schools follow the school page's states; ISI schools
|
||||
read "Inspected by ISI"; schools with no data keep "No inspection in our
|
||||
dataset".
|
||||
|
||||
**Backend.** The list payload and the detail `school_info` gain `inspectorate`
|
||||
and `ofsted_listed`. FE detail needs no new block: `_ofsted_block` already
|
||||
carries `current_grade`, `latest_visit`, the report-card date and `framework`.
|
||||
`inspectorate` and `ofsted_listed` join `_EXTRA_OPTIONAL_COLUMNS` in
|
||||
`backend/data_loader.py`, so a `dim_school` built before PR 1 makes the main
|
||||
query retry without them (no badge for groups 4–5) instead of failing.
|
||||
|
||||
Copy follows the site's factual rule: inspectorate names and Ofsted's own terms
|
||||
("Short inspection", "Not yet inspected"), no derived sentences.
|
||||
|
||||
## Delivery
|
||||
|
||||
Two PRs, as for C1 and C2.
|
||||
|
||||
**PR 1 — pipeline:** the GIAS fields, the two FE streams and their selection
|
||||
module, the staging and intermediate models, `dim_school`'s two columns, the
|
||||
DAG selector, `OPTIONAL_PARENTS`, unit and data tests.
|
||||
|
||||
**PR 2 — site (after the DAGs have run on PR 1):** the backend fields and the
|
||||
optional-column fallback, the badge rule, the school-page states, compare, and
|
||||
E2E.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Extractor (pytest, pandas-only module):** from saved page HTML, the newest
|
||||
"as at" file and the latest cumulative file per page are chosen and
|
||||
monitoring files are ignored; the doubled-underscore file name parses; real
|
||||
headers map to the declared fields.
|
||||
- **dbt unit tests:** previous-framework grade only; grade with a later short
|
||||
inspection (grade dated by the full inspection, latest visit `short`); grade
|
||||
with a newer report card (report card, no grade); report card only;
|
||||
UKPRN-only match; a school record wins over an FE record; a listed,
|
||||
uninspected school is in `int_ofsted_listed` and not in `int_ofsted_latest`.
|
||||
- **Backend pytest:** both new fields in the list and detail payloads; the main
|
||||
query falls back when either column is missing.
|
||||
- **Jest:** one case per badge rule including "no badge"; every badge caller
|
||||
renders nothing when there is no badge; the school page's FE, ISI and
|
||||
listed-uninspected states; compare cells.
|
||||
- **E2E:** Queen Mary's College (145175) shows an Ofsted grade, not "Not yet
|
||||
inspected"; Abbey College Cambridge (136083) shows "Inspected by ISI"; Abbas
|
||||
and Templecombe Church School (150031) shows "Not yet inspected".
|
||||
|
||||
## Rollout and verification
|
||||
|
||||
1. Locally, load real GIAS, school MI and FE MI through the changed extractors,
|
||||
build the models, and check: about 490 FE rows (443 + 45 + 2); 145175 graded
|
||||
2 on 2022-12-06; 143526 report card 2026-05-12, safeguarding met; 1,341
|
||||
schools with inspectorate "ISI"; 150031 listed, not in `fact_ofsted_latest`.
|
||||
2. PR 1 merges; Tudor runs the daily DAG (GIAS fields) and the monthly Ofsted
|
||||
DAG on staging; the staging API shows the same examples.
|
||||
3. PR 2 merges, so its E2E journeys run against loaded data.
|
||||
4. Production, Tudor's decision: promote PR 1, run both DAGs, promote PR 2. If
|
||||
PR 2 arrives first, the fallback query runs and groups 4–5 show no badge
|
||||
until the DAGs have run.
|
||||
5. Validate on production: every "Not yet inspected" belongs to a school
|
||||
Ofsted's own data lists without an inspection; every "Inspected by ISI"
|
||||
matches GIAS; every FE grade, date and report card matches Ofsted's FE files;
|
||||
schools with no data show no badge. Then mark H4 resolved in the audit
|
||||
report.
|
||||
|
||||
## Risks
|
||||
|
||||
- **FE page or file naming changes.** Selection is by title prefix and file-name
|
||||
pattern; if nothing matches, the FE streams emit nothing and FE schools fall
|
||||
back to "no badge", never a false claim. The extractor tests pin the patterns
|
||||
against real HTML.
|
||||
- **A provider's UKPRN shared by several URNs.** Each matched URN takes the
|
||||
provider's result; the URN match is always tried first.
|
||||
- **"Not yet inspected" disappears briefly** if PR 2 is promoted before the
|
||||
DAGs run (fallback query). It never appears wrongly.
|
||||
|
||||
## Amendments (implementation plan, 2026-10-13)
|
||||
|
||||
- **UKPRN comes from a new view, `stg_gias_ukprn`,** on the raw GIAS source,
|
||||
not from a `ukprn` column on `stg_gias_establishments`. A `ref` to
|
||||
`stg_gias_establishments` from the Ofsted models would put them in the daily
|
||||
build's `stg_gias_establishments+` selection, where their Ofsted parents are
|
||||
not built (`pipeline/tests/test_dag_selectors.py`). The monthly Ofsted build
|
||||
selects `stg_gias_ukprn+`; the daily build must not rebuild it, because
|
||||
rebuilding a view drops the views on it (CASCADE) and `dim_school` reads
|
||||
them. `stg_gias_establishments` gains only `inspectorate_name`.
|
||||
- **First rollout order (each environment, once).** The new models need
|
||||
`raw.gias_establishments."UKPRN"`, which only a daily run with PR 1 creates,
|
||||
and `dim_school`'s unit tests need `int_ofsted_listed`, which only a monthly
|
||||
run creates. So: run the daily DAG (its `dbt_build` fails once on
|
||||
`dim_school`'s unit tests — expected), then `school_data_monthly_ofsted`,
|
||||
then the daily DAG again. From then on both schedules run normally.
|
||||
- **A report card is recognised by `rc_inspection_date` as well as by its area
|
||||
labels** (`hasReportCard`), on the school page and in compare. An FE report
|
||||
card carries a safeguarding result and no school areas, so the label test
|
||||
alone read it as "No overall grade".
|
||||
- **The compare page's Ofsted link is fixed.** `ofsted_page_url` built
|
||||
`https://reports.ofsted.gov.uk/provider/21/{urn}`; category 21 is primary
|
||||
schools, so the link 404s for every secondary, special and FE school
|
||||
(checked: 137086, 101099, 130457, 145175). It now uses
|
||||
`…/inspection-reports/find-inspection-report/provider/ELS/{urn}`, which
|
||||
redirects to the right category and is what the school page already uses.
|
||||
@@ -554,6 +554,49 @@ test('a grade confirmed at an ungraded visit says so and is dated by it', async
|
||||
await expect(section.getByText('Confirmed at an ungraded inspection, 18 July 2024')).toBeVisible();
|
||||
});
|
||||
|
||||
test('a further education college shows its Ofsted result, not "Not yet inspected"', async ({ page }) => {
|
||||
// Queen Mary's College (145175) is inspected under Ofsted's further education
|
||||
// framework, which the site did not load: its badge said "Not yet inspected"
|
||||
// (audit H4). Its result may change; the label must never be absent or wrong.
|
||||
const res = await page.request.get('/api/schools/145175');
|
||||
expect(res.ok()).toBeTruthy();
|
||||
await page.goto('/school/145175');
|
||||
const ofsted = page.locator('#ofsted');
|
||||
await expect(ofsted).toBeVisible();
|
||||
await expect(ofsted.getByText('Further education and skills')).toBeVisible();
|
||||
await expect(ofsted).not.toContainText('Not yet inspected');
|
||||
});
|
||||
|
||||
test('an ISI-inspected school says so instead of "Not yet inspected"', async ({ page }) => {
|
||||
const res = await page.request.get('/api/schools?school_type=independent&phase=secondary&page_size=100');
|
||||
expect(res.ok()).toBeTruthy();
|
||||
const school = ((await res.json()).schools ?? []).find(
|
||||
(s: { inspectorate?: string | null; ofsted_date?: string | null; ofsted_rc_date?: string | null }) =>
|
||||
s.inspectorate === 'ISI' && !s.ofsted_date && !s.ofsted_rc_date);
|
||||
test.skip(!school, 'no ISI school without an Ofsted record here');
|
||||
await page.goto(`/school/${school.urn}`);
|
||||
await expect(page.getByText('Inspected by the Independent Schools Inspectorate (ISI)')).toBeVisible();
|
||||
await expect(page.locator('#ofsted')).not.toContainText('Not yet inspected');
|
||||
});
|
||||
|
||||
test('"Not yet inspected" appears only for a school Ofsted lists without an inspection', async ({ page }) => {
|
||||
const res = await page.request.get('/api/schools?phase=primary&page_size=500');
|
||||
expect(res.ok()).toBeTruthy();
|
||||
type Row = { urn: number; inspectorate?: string | null; ofsted_listed?: boolean | null; ofsted_date?: string | null; ofsted_rc_date?: string | null; ofsted_grade?: number | null };
|
||||
const schools: Row[] = (await res.json()).schools ?? [];
|
||||
// No Ofsted record, and not an ISI school (whose badge names ISI instead).
|
||||
const uninspected = (s: Row) => !s.ofsted_date && !s.ofsted_rc_date && !s.ofsted_grade && s.inspectorate !== 'ISI';
|
||||
const unlisted = schools.find(s => !s.ofsted_listed && uninspected(s));
|
||||
const listed = schools.find(s => s.ofsted_listed && uninspected(s));
|
||||
test.skip(!listed, 'no listed, uninspected primary on this page');
|
||||
await page.goto(`/school/${listed!.urn}`);
|
||||
await expect(page.locator('#ofsted').getByText('Not yet inspected')).toBeVisible();
|
||||
if (unlisted) {
|
||||
await page.goto(`/school/${unlisted.urn}`);
|
||||
await expect(page.getByText('Not yet inspected')).toHaveCount(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('school detail page renders name and performance data', async ({ page }) => {
|
||||
await searchByName(page, 'primary');
|
||||
const firstSchool = schoolLinks(page).first();
|
||||
|
||||
@@ -69,6 +69,18 @@ const data: Record<string, ComparisonData> = {
|
||||
};
|
||||
|
||||
describe('CompareOfsted', () => {
|
||||
it('words a school without an Ofsted record from the evidence (audit H4)', () => {
|
||||
const isi = { ...school(4, 'ISI School'), inspectorate: 'ISI' } as School;
|
||||
const listed = { ...school(5, 'New Academy'), ofsted_listed: true } as School;
|
||||
render(<CompareOfsted schools={[isi, listed]} data={{
|
||||
'4': { school_info: isi, yearly_data: [] } as unknown as ComparisonData,
|
||||
'5': { school_info: listed, yearly_data: [] } as unknown as ComparisonData,
|
||||
}} />);
|
||||
expect(screen.getAllByText('Inspected by ISI').length).toBeGreaterThan(0);
|
||||
expect(screen.getAllByText('Not yet inspected').length).toBeGreaterThan(0);
|
||||
expect(screen.queryByText(/No inspection outcome in our dataset/)).toBeNull();
|
||||
});
|
||||
|
||||
it('renders the three regimes without inventing an overall grade for report cards', () => {
|
||||
render(<CompareOfsted schools={schools} data={data} />);
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { InspectionNotice } from '@/components/school/InspectionNotice';
|
||||
|
||||
it('names ISI and links its reports for an ISI school (audit H4)', () => {
|
||||
render(<InspectionNotice school={{ urn: 136083, inspectorate: 'ISI', ofsted_listed: false }} />);
|
||||
expect(screen.getByText('Inspected by the Independent Schools Inspectorate (ISI)')).toBeInTheDocument();
|
||||
expect(screen.getByRole('link', { name: /ISI reports/ })).toHaveAttribute('href', 'https://www.isi.net/reports/');
|
||||
});
|
||||
|
||||
it('says "Not yet inspected" for a school Ofsted lists without an inspection', () => {
|
||||
render(<InspectionNotice school={{ urn: 150031, inspectorate: 'Not applicable', ofsted_listed: true }} />);
|
||||
expect(screen.getByText('Not yet inspected')).toBeInTheDocument();
|
||||
expect(screen.getByRole('link', { name: /Ofsted reports/ })).toHaveAttribute(
|
||||
'href', 'https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/150031');
|
||||
});
|
||||
|
||||
it('renders nothing without inspection data, including an unknown listing', () => {
|
||||
const { container } = render(<InspectionNotice school={{ urn: 1, inspectorate: null, ofsted_listed: null }} />);
|
||||
expect(container).toBeEmptyDOMElement();
|
||||
});
|
||||
@@ -82,3 +82,14 @@ it('prints an ungraded outcome when there is no grade (Oakgrove)', () => {
|
||||
expect(screen.getByText('No overall grade')).toBeInTheDocument();
|
||||
expect(screen.getByText(/Ungraded inspection, 13 November 2024: Standards maintained/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('labels a further education record and prints its short inspection (audit H4)', () => {
|
||||
renderSection({
|
||||
framework: 'Further education and skills',
|
||||
current_grade: { grade: 2, date: '2019-03-05', basis: 'graded' },
|
||||
latest_visit: { date: '2023-02-01', kind: 'short', outcome: null },
|
||||
});
|
||||
expect(screen.getByText('Further education and skills')).toBeInTheDocument();
|
||||
expect(screen.getByText('Good')).toBeInTheDocument();
|
||||
expect(screen.getByText('Latest visit: Short inspection, 1 February 2023')).toBeInTheDocument();
|
||||
});
|
||||
@@ -52,3 +52,8 @@ describe('SchoolRow shares the school page flags', () => {
|
||||
expect(screen.getByText("Boys' school")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('renders no Ofsted badge for a school with no inspection data (audit H4)', () => {
|
||||
const { container } = render(<SchoolRow school={{ ...base, ofsted_grade: null, ofsted_date: null, ofsted_listed: false, inspectorate: null }} />);
|
||||
expect(container.textContent).not.toMatch(/Not yet inspected|Inspected/);
|
||||
});
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
summariseAdmissions,
|
||||
summariseReportCard,
|
||||
verdict,
|
||||
noInspectionLabel,
|
||||
} from '@/lib/compareLogic';
|
||||
import type { OfstedInspection, SchoolAdmissions } from '@/lib/types';
|
||||
|
||||
@@ -359,3 +360,14 @@ describe('basket limits per comparison group', () => {
|
||||
expect(kept).toEqual([0, 1, 2, 3, 4, 7, 8, 9]);
|
||||
});
|
||||
});
|
||||
|
||||
it('shows an FE report card with only a safeguarding result as a report card (audit H4)', () => {
|
||||
const display = ofstedDisplay(ofsted({ rc_inspection_date: '2026-05-12', rc_safeguarding_met: true, report_card: {} }));
|
||||
expect(display.kind).toBe('report_card');
|
||||
});
|
||||
|
||||
it('words the no-record compare cell from the evidence', () => {
|
||||
expect(noInspectionLabel({ inspectorate: 'ISI' })).toBe('Inspected by ISI');
|
||||
expect(noInspectionLabel({ ofsted_listed: true })).toBe('Not yet inspected');
|
||||
expect(noInspectionLabel({ ofsted_listed: null })).toBe('No inspection in our dataset');
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { gradeSourceLine, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
|
||||
import { gradeSourceLine, hasReportCard, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
|
||||
import type { OfstedInspection } from '@/lib/types';
|
||||
|
||||
const base = { report_card: {} } as unknown as OfstedInspection;
|
||||
@@ -45,3 +45,12 @@ describe('showLatestVisitLine', () => {
|
||||
latest_visit: { date: '2025-06-17', kind: 'graded', outcome: null } })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it('names a short inspection as Ofsted does', () => {
|
||||
expect(latestVisitLine({ date: '2023-02-01', kind: 'short', outcome: null })).toBe('Short inspection, 1 February 2023');
|
||||
});
|
||||
|
||||
it('treats a report card with only a safeguarding result as a report card (FE)', () => {
|
||||
expect(hasReportCard({ rc_inspection_date: '2026-05-12', report_card: {} })).toBe(true);
|
||||
expect(hasReportCard({ rc_inspection_date: null, report_card: {} })).toBe(false);
|
||||
});
|
||||
@@ -148,40 +148,40 @@ describe('ofstedLegacyAreas', () => {
|
||||
describe('buildOfstedListBadge', () => {
|
||||
it('returns grade word + year for OEIF Outstanding', () => {
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: 1, ofsted_grade_date: '2023-11-15', ofsted_date: '2023-11-15', ofsted_framework: 'OEIF' });
|
||||
expect(badge.label).toBe('Outstanding · 2023');
|
||||
expect(badge.cssClass).toBe('ofsted1');
|
||||
expect(badge!.label).toBe('Outstanding · 2023');
|
||||
expect(badge!.cssClass).toBe('ofsted1');
|
||||
});
|
||||
|
||||
it('returns grade word for each OEIF grade', () => {
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2022-05-01' }).label).toBe('Good · 2022');
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 3, ofsted_grade_date: '2021-01-01' }).label).toBe('Req. Improvement · 2021');
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 4, ofsted_grade_date: '2020-03-01' }).label).toBe('Inadequate · 2020');
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2022-05-01' })!.label).toBe('Good · 2022');
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 3, ofsted_grade_date: '2021-01-01' })!.label).toBe('Req. Improvement · 2021');
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 4, ofsted_grade_date: '2020-03-01' })!.label).toBe('Inadequate · 2020');
|
||||
});
|
||||
|
||||
it('prints a grade without a year when its date is missing', () => {
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: null, ofsted_date: '2025-01-01' });
|
||||
expect(badge.label).toBe('Good');
|
||||
expect(badge.cssClass).toBe('ofsted2');
|
||||
expect(badge!.label).toBe('Good');
|
||||
expect(badge!.cssClass).toBe('ofsted2');
|
||||
});
|
||||
|
||||
it('dates a grade by the inspection that awarded it, not the latest visit', () => {
|
||||
// Washwood Heath: Good from a March 2020 graded inspection; latest visit
|
||||
// an ungraded one in May 2025 (audit M1).
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2020-03-03', ofsted_date: '2025-05-21' });
|
||||
expect(badge.label).toBe('Good · 2020');
|
||||
expect(badge!.label).toBe('Good · 2020');
|
||||
});
|
||||
|
||||
it('shows Inspected for a latest inspection that gave no grade (Rabbsfarm)', () => {
|
||||
// Audit C1: this used to read "Good · 2025", from a 2020 ungraded visit.
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_grade_date: null, ofsted_date: '2025-06-17' });
|
||||
expect(badge.label).toBe('Inspected · 2025');
|
||||
expect(badge.cssClass).toBe('ofstedInspected');
|
||||
expect(badge!.label).toBe('Inspected · 2025');
|
||||
expect(badge!.cssClass).toBe('ofstedInspected');
|
||||
});
|
||||
|
||||
it('returns a Report Card badge when ofsted_rc_date is present', () => {
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_rc_date: '2026-02-03' });
|
||||
expect(badge.label).toBe('Report Card · 2026');
|
||||
expect(badge.cssClass).toBe('ofstedRc');
|
||||
expect(badge!.label).toBe('Report Card · 2026');
|
||||
expect(badge!.cssClass).toBe('ofstedRc');
|
||||
});
|
||||
|
||||
it('a report card wins over a carried-forward legacy grade', () => {
|
||||
@@ -194,34 +194,46 @@ describe('buildOfstedListBadge', () => {
|
||||
ofsted_framework: 'Schools - S5',
|
||||
ofsted_rc_date: '2026-02-03',
|
||||
});
|
||||
expect(badge.label).toBe('Report Card · 2026');
|
||||
expect(badge.cssClass).toBe('ofstedRc');
|
||||
expect(badge!.label).toBe('Report Card · 2026');
|
||||
expect(badge!.cssClass).toBe('ofstedRc');
|
||||
});
|
||||
|
||||
it('returns an "Inspected" badge for an OEIF inspection with no overall grade (post-Sept-2024)', () => {
|
||||
// Inspected after Sept 2024: inspection on record (date + framework) but
|
||||
// Ofsted no longer issues an overall grade. Must NOT read as "Not yet inspected".
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: '2024-11-01', ofsted_framework: 'OEIF' });
|
||||
expect(badge.label).toBe('Inspected · 2024');
|
||||
expect(badge.cssClass).toBe('ofstedInspected');
|
||||
expect(badge!.label).toBe('Inspected · 2024');
|
||||
expect(badge!.cssClass).toBe('ofstedInspected');
|
||||
});
|
||||
|
||||
it('returns an "Inspected" badge without a year when the grade is missing and date is absent but a record exists', () => {
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_framework: 'OEIF' });
|
||||
expect(badge.label).toBe('Inspected');
|
||||
expect(badge.cssClass).toBe('ofstedInspected');
|
||||
expect(badge!.label).toBe('Inspected');
|
||||
expect(badge!.cssClass).toBe('ofstedInspected');
|
||||
});
|
||||
|
||||
it('returns pending badge when no grade and no inspection on record', () => {
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_framework: null });
|
||||
expect(badge.label).toBe('Not yet inspected');
|
||||
expect(badge.cssClass).toBe('ofstedPending');
|
||||
it('says "Not yet inspected" only when Ofsted lists the school uninspected (audit H4)', () => {
|
||||
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_listed: true });
|
||||
expect(badge).toEqual({ label: 'Not yet inspected', cssClass: 'ofstedPending' });
|
||||
});
|
||||
|
||||
it('returns pending badge when all fields are undefined', () => {
|
||||
const badge = buildOfstedListBadge({});
|
||||
expect(badge.label).toBe('Not yet inspected');
|
||||
expect(badge.cssClass).toBe('ofstedPending');
|
||||
it('names ISI for a school GIAS says ISI inspects', () => {
|
||||
expect(buildOfstedListBadge({ ofsted_date: null, inspectorate: 'ISI', ofsted_listed: false }))
|
||||
.toEqual({ label: 'Inspected by ISI', cssClass: 'ofstedPending' });
|
||||
});
|
||||
|
||||
it('an Ofsted record wins over ISI', () => {
|
||||
expect(buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2022-03-01', inspectorate: 'ISI' })!.label)
|
||||
.toBe('Good · 2022');
|
||||
});
|
||||
|
||||
it('gives no badge without any inspection data', () => {
|
||||
expect(buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_framework: null })).toBeNull();
|
||||
expect(buildOfstedListBadge({ ofsted_listed: null, inspectorate: 'Not applicable' })).toBeNull();
|
||||
});
|
||||
|
||||
it('gives no badge when all fields are undefined (audit H4)', () => {
|
||||
expect(buildOfstedListBadge({})).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ function cardHtml(school: School, ctx: CardContext): string {
|
||||
? `${metric ? ' · ' : ''}${school.total_pupils.toLocaleString('en-GB')} pupils` : '';
|
||||
return `<div class="sc-card">
|
||||
<div class="sc-card-head"><a class="sc-card-name" href="${href}">${escapeHtml(school.school_name)}</a>${distance}</div>
|
||||
<div class="sc-card-tags"><span class="sc-ofsted sc-${badge.cssClass}">${escapeHtml(badge.label)}</span>${type}</div>
|
||||
<div class="sc-card-tags">${badge ? `<span class="sc-ofsted sc-${badge.cssClass}">${escapeHtml(badge.label)}</span>` : ''}${type}</div>
|
||||
${metric || pupils ? `<div class="sc-card-metric">${metric}${pupils}</div>` : ''}
|
||||
<div class="sc-card-actions">
|
||||
<a class="btn btn-tertiary btn-sm" href="${href}">View</a>
|
||||
|
||||
@@ -66,9 +66,11 @@ export function SchoolRow({
|
||||
<a href={schoolUrl(school.urn, school.school_name)} className={styles.schoolName}>
|
||||
{school.school_name}
|
||||
</a>
|
||||
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
|
||||
{ofstedBadge.label}
|
||||
</span>
|
||||
{ofstedBadge && (
|
||||
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
|
||||
{ofstedBadge.label}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Line 2: Context tags */}
|
||||
|
||||
@@ -67,9 +67,11 @@ export function SecondarySchoolRow({
|
||||
<a href={schoolUrl(school.urn, school.school_name)} className={styles.schoolName}>
|
||||
{school.school_name}
|
||||
</a>
|
||||
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
|
||||
{ofstedBadge.label}
|
||||
</span>
|
||||
{ofstedBadge && (
|
||||
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
|
||||
{ofstedBadge.label}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Line 2: Context tags */}
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
summariseAdmissions,
|
||||
verdict,
|
||||
type ReportCardSummary,
|
||||
noInspectionLabel,
|
||||
} from '@/lib/compareLogic';
|
||||
import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types';
|
||||
import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared';
|
||||
@@ -91,7 +92,7 @@ export function CompareAtAGlance({
|
||||
No overall grade
|
||||
</span>
|
||||
)}
|
||||
{display.kind === 'none' && <span className={s.small}>No inspection in our dataset</span>}
|
||||
{display.kind === 'none' && <span className={s.small}>{noInspectionLabel(data[String(school.urn)]?.school_info ?? school)}</span>}
|
||||
</Cell>
|
||||
);
|
||||
})}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
ofstedDisplay,
|
||||
rcAreaLabel,
|
||||
type OfstedDisplay,
|
||||
noInspectionLabel,
|
||||
} from '@/lib/compareLogic';
|
||||
import { gradeSourceLine } from '@/lib/ofstedStatus';
|
||||
import type { ComparisonData, OfstedCurrentGrade, OfstedInspection, School } from '@/lib/types';
|
||||
@@ -41,9 +42,11 @@ function yearsSince(iso: string | null): number | null {
|
||||
return (Date.now() - d.getTime()) / (365.25 * 24 * 3600 * 1000);
|
||||
}
|
||||
|
||||
function ResultCell({ display, current }: { display: OfstedDisplay; current: OfstedCurrentGrade | null }) {
|
||||
function ResultCell({ display, current, school }: {
|
||||
display: OfstedDisplay; current: OfstedCurrentGrade | null; school: School;
|
||||
}) {
|
||||
if (display.kind === 'none') {
|
||||
return <span className={s.small}>No inspection outcome in our dataset</span>;
|
||||
return <span className={s.small}>{noInspectionLabel(school)}</span>;
|
||||
}
|
||||
if (display.kind === 'report_card') {
|
||||
return (
|
||||
@@ -175,7 +178,7 @@ export function CompareOfsted({
|
||||
<Measure label="Result">
|
||||
{schools.map((school, i) => (
|
||||
<Cell key={school.urn} school={school} index={i}>
|
||||
<ResultCell display={displays[i]} current={data[String(school.urn)]?.ofsted?.current_grade ?? null} />
|
||||
<ResultCell display={displays[i]} current={data[String(school.urn)]?.ofsted?.current_grade ?? null} school={data[String(school.urn)]?.school_info ?? school} />
|
||||
</Cell>
|
||||
))}
|
||||
</Measure>
|
||||
@@ -213,7 +216,7 @@ export function CompareOfsted({
|
||||
schoolName={school.school_name}
|
||||
/>
|
||||
) : (
|
||||
<span className={s.small}>No inspection in our dataset</span>
|
||||
<span className={s.small}>{noInspectionLabel(data[String(school.urn)]?.school_info ?? school)}</span>
|
||||
)}
|
||||
</Cell>
|
||||
);
|
||||
@@ -228,7 +231,7 @@ export function CompareOfsted({
|
||||
{schools.map((school, i) => {
|
||||
const url =
|
||||
data[String(school.urn)]?.ofsted?.ofsted_page_url ??
|
||||
`https://reports.ofsted.gov.uk/provider/21/${school.urn}`;
|
||||
`https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/${school.urn}`;
|
||||
return (
|
||||
<Cell key={school.urn} school={school} index={i}>
|
||||
{/* Short visible label to save space on mobile (the coloured
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* InspectionNotice — the school page's inspection section when there is no
|
||||
* Ofsted inspection record (audit H4): the inspectorate GIAS names (ISI), or
|
||||
* Ofsted's own "not yet inspected" when its data lists the school. Nothing
|
||||
* otherwise: no claim the data cannot support. Server component.
|
||||
*/
|
||||
|
||||
import type { School } from '@/lib/types';
|
||||
import { Section, sectionStyles as styles } from './sectionShared';
|
||||
|
||||
const OFSTED_REPORTS = 'https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/';
|
||||
const ISI_REPORTS = 'https://www.isi.net/reports/';
|
||||
|
||||
export function InspectionNotice({ school }: { school: Pick<School, 'urn' | 'inspectorate' | 'ofsted_listed'> }) {
|
||||
if (school.inspectorate === 'ISI') {
|
||||
return (
|
||||
<Section id="ofsted">
|
||||
<h2 className={styles.sectionTitle}>
|
||||
Inspection
|
||||
<a href={ISI_REPORTS} target="_blank" rel="noopener noreferrer" className={styles.ofstedReportLink}
|
||||
data-umami-event="external_link_clicked" data-umami-event-target="isi">
|
||||
ISI reports ↗
|
||||
</a>
|
||||
</h2>
|
||||
<p className={styles.ofstedDisclaimer}>Inspected by the Independent Schools Inspectorate (ISI)</p>
|
||||
</Section>
|
||||
);
|
||||
}
|
||||
if (school.ofsted_listed) {
|
||||
return (
|
||||
<Section id="ofsted">
|
||||
<h2 className={styles.sectionTitle}>
|
||||
Ofsted Rating
|
||||
<a href={`${OFSTED_REPORTS}${school.urn}`} target="_blank" rel="noopener noreferrer"
|
||||
className={styles.ofstedReportLink} data-umami-event="external_link_clicked" data-umami-event-target="ofsted">
|
||||
Ofsted reports ↗
|
||||
</a>
|
||||
</h2>
|
||||
<div className={styles.ofstedHeader}>
|
||||
<span className={styles.ofstedGrade}>Not yet inspected</span>
|
||||
</div>
|
||||
</Section>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import { ofstedDisplay } from '@/lib/compareLogic';
|
||||
import { formatOfstedDate, gradeSourceLine, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
|
||||
import { FE_FRAMEWORK, formatOfstedDate, gradeSourceLine, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
|
||||
import type { OfstedInspection } from '@/lib/types';
|
||||
import { Section, sectionStyles as styles } from './sectionShared';
|
||||
|
||||
@@ -71,6 +71,10 @@ export function OfstedSection({
|
||||
</a>
|
||||
</h2>
|
||||
|
||||
{ofsted.framework === FE_FRAMEWORK && (
|
||||
<p className={styles.ofstedDisclaimer}>{FE_FRAMEWORK}</p>
|
||||
)}
|
||||
|
||||
{isReportCard ? (
|
||||
/* ── New Report Card layout ── */
|
||||
<>
|
||||
|
||||
@@ -19,6 +19,8 @@ import type {
|
||||
import { ofstedLegacyAreas } from '@/lib/utils';
|
||||
import type { SchoolFlags } from '@/lib/schoolSections';
|
||||
import { OfstedSection } from './OfstedSection';
|
||||
import { InspectionNotice } from './InspectionNotice';
|
||||
import { hasReportCard } from '@/lib/ofstedStatus';
|
||||
import { ResultsSection } from './ResultsSection';
|
||||
import { AdmissionsSection } from './AdmissionsSection';
|
||||
import { DistanceSection } from './DistanceSection';
|
||||
@@ -54,7 +56,8 @@ export function PrimarySchoolSections({
|
||||
const primaryAvg = nationalAvg?.primary ?? {};
|
||||
const secondaryAvg = nationalAvg?.secondary ?? {};
|
||||
|
||||
const isReportCard = !!(ofsted?.report_card && Object.keys(ofsted.report_card).length > 0);
|
||||
// By date as well as area labels: an FE report card has only a safeguarding result (audit H4).
|
||||
const isReportCard = !!ofsted && hasReportCard(ofsted);
|
||||
// Report cards are dated by their own inspection (rc_inspection_date);
|
||||
// anything else by the school's latest visit, never the older inspection a
|
||||
// grade may come from.
|
||||
@@ -71,7 +74,7 @@ export function PrimarySchoolSections({
|
||||
|
||||
return (
|
||||
<>
|
||||
{ofsted && (
|
||||
{ofsted ? (
|
||||
<OfstedSection
|
||||
ofsted={ofsted}
|
||||
urn={schoolInfo.urn}
|
||||
@@ -80,7 +83,7 @@ export function PrimarySchoolSections({
|
||||
oeifAllSameGrade={oeifAllSameGrade}
|
||||
oeifAreas={oeifAreas}
|
||||
/>
|
||||
)}
|
||||
) : <InspectionNotice school={schoolInfo} />}
|
||||
|
||||
{flags.hasAnyResults && flags.latestResults && (
|
||||
<ResultsSection
|
||||
|
||||
@@ -19,6 +19,8 @@ import type {
|
||||
import { ofstedLegacyAreas } from '@/lib/utils';
|
||||
import type { SecondaryFlags } from '@/lib/schoolSections';
|
||||
import { OfstedSection } from './OfstedSection';
|
||||
import { InspectionNotice } from './InspectionNotice';
|
||||
import { hasReportCard } from '@/lib/ofstedStatus';
|
||||
import { GcseSection } from './GcseSection';
|
||||
import { DestinationsSection } from './DestinationsSection';
|
||||
import { Post16DestinationsSection } from './Post16DestinationsSection';
|
||||
@@ -60,7 +62,8 @@ export function SecondarySchoolSections({
|
||||
}: SecondarySchoolSectionsProps) {
|
||||
const secondaryAvg = nationalAvg?.secondary ?? {};
|
||||
|
||||
const isReportCard = !!(ofsted?.report_card && Object.keys(ofsted.report_card).length > 0);
|
||||
// By date as well as area labels: an FE report card has only a safeguarding result (audit H4).
|
||||
const isReportCard = !!ofsted && hasReportCard(ofsted);
|
||||
const ofstedInspectedDate = isReportCard
|
||||
? ofsted?.rc_inspection_date ?? null
|
||||
: ofsted?.latest_visit?.date ?? null;
|
||||
@@ -74,7 +77,7 @@ export function SecondarySchoolSections({
|
||||
|
||||
return (
|
||||
<div className={styles.secondaryScope}>
|
||||
{ofsted && (
|
||||
{ofsted ? (
|
||||
<OfstedSection
|
||||
ofsted={ofsted}
|
||||
urn={schoolInfo.urn}
|
||||
@@ -83,7 +86,7 @@ export function SecondarySchoolSections({
|
||||
oeifAllSameGrade={oeifAllSameGrade}
|
||||
oeifAreas={oeifAreas}
|
||||
/>
|
||||
)}
|
||||
) : <InspectionNotice school={schoolInfo} />}
|
||||
|
||||
{flags.hasResults && flags.latestResults && (
|
||||
<GcseSection
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
* methodology instead of thresholding point estimates.
|
||||
*/
|
||||
|
||||
import type { OfstedInspection, SchoolAdmissions } from './types';
|
||||
import { hasReportCard } from './ofstedStatus';
|
||||
import type { OfstedInspection, School, SchoolAdmissions } from './types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Verdicts against an anchor (England average or state-school benchmark)
|
||||
@@ -104,6 +105,16 @@ export type OfstedDisplay =
|
||||
| { kind: 'no_overall_grade' }
|
||||
| { kind: 'report_card'; summary: ReportCardSummary };
|
||||
|
||||
/**
|
||||
* The compare cell when a school has no Ofsted record (audit H4): ISI per GIAS,
|
||||
* Ofsted's own "not yet inspected", or plainly that we hold nothing.
|
||||
*/
|
||||
export function noInspectionLabel(school: Pick<School, 'inspectorate' | 'ofsted_listed'>): string {
|
||||
if (school.inspectorate === 'ISI') return 'Inspected by ISI';
|
||||
if (school.ofsted_listed) return 'Not yet inspected';
|
||||
return 'No inspection in our dataset';
|
||||
}
|
||||
|
||||
export function ofstedDisplay(
|
||||
ofsted: OfstedInspection | null | undefined,
|
||||
): OfstedDisplay {
|
||||
@@ -111,7 +122,7 @@ export function ofstedDisplay(
|
||||
|
||||
// A report card is the newest inspection format; when present it wins —
|
||||
// never derive or prefer an overall grade alongside it.
|
||||
if (ofsted.report_card && Object.keys(ofsted.report_card).length > 0) {
|
||||
if (hasReportCard(ofsted)) {
|
||||
return { kind: 'report_card', summary: summariseReportCard(ofsted) };
|
||||
}
|
||||
|
||||
|
||||
@@ -17,8 +17,22 @@ const VISIT_KIND: Record<OfstedLatestVisit['kind'], string> = {
|
||||
report_card: 'Report card inspection',
|
||||
graded: 'Graded inspection',
|
||||
ungraded: 'Ungraded inspection',
|
||||
short: 'Short inspection',
|
||||
};
|
||||
|
||||
/** fact_ofsted_latest.framework for Ofsted's further education and skills records (audit H4). */
|
||||
export const FE_FRAMEWORK = 'Further education and skills';
|
||||
|
||||
/**
|
||||
* A report card is in force. Checked by its date as well as its area labels:
|
||||
* an FE report card carries a safeguarding result and no school areas.
|
||||
*/
|
||||
export function hasReportCard(
|
||||
ofsted: Pick<OfstedInspection, 'rc_inspection_date' | 'report_card'>,
|
||||
): boolean {
|
||||
return !!ofsted.rc_inspection_date || Object.keys(ofsted.report_card ?? {}).length > 0;
|
||||
}
|
||||
|
||||
/** "Graded inspection, 6 July 2016" or "Confirmed at an ungraded inspection, 14 March 2023". */
|
||||
export function gradeSourceLine(current: OfstedCurrentGrade): string {
|
||||
const when = formatOfstedDate(current.date);
|
||||
|
||||
@@ -86,6 +86,10 @@ export interface School {
|
||||
/** The school's latest inspection of any kind. */
|
||||
ofsted_date?: string | null;
|
||||
ofsted_framework?: string | null;
|
||||
/** The inspectorate GIAS names ("ISI", "Ofsted", "Not applicable"). Audit H4. */
|
||||
inspectorate?: string | null;
|
||||
/** Whether Ofsted's own data lists the school, inspected or not. Audit H4. */
|
||||
ofsted_listed?: boolean | null;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -93,7 +97,8 @@ export interface School {
|
||||
// ============================================================================
|
||||
|
||||
export interface OfstedInspection {
|
||||
framework: 'OEIF' | 'ReportCard' | null;
|
||||
/** Ofsted's event grouping (e.g. "Schools - S5"), or "Further education and skills" for FE records (audit H4). */
|
||||
framework: string | null;
|
||||
inspection_date: string | null;
|
||||
/** Start date of the report-card inspection itself (Nov 2025+); null otherwise. */
|
||||
rc_inspection_date?: string | null;
|
||||
@@ -143,7 +148,7 @@ export interface OfstedCurrentGrade {
|
||||
|
||||
export interface OfstedLatestVisit {
|
||||
date: string;
|
||||
kind: 'report_card' | 'graded' | 'ungraded';
|
||||
kind: 'report_card' | 'graded' | 'ungraded' | 'short';
|
||||
outcome: string | null;
|
||||
}
|
||||
|
||||
|
||||
@@ -678,7 +678,9 @@ export interface OfstedListBadge {
|
||||
* or an ungraded visit whose outcome names no grade): "Inspected · YYYY",
|
||||
* dated by the latest visit, so a school never reads as both inspected and
|
||||
* "Not yet inspected"
|
||||
* - No inspection on record: "Not yet inspected" in grey
|
||||
* - No inspection on record (audit H4): "Inspected by ISI" when GIAS names
|
||||
* ISI; "Not yet inspected" only when Ofsted's own data lists the school;
|
||||
* otherwise no badge, never a claim the data cannot support.
|
||||
*/
|
||||
export function buildOfstedListBadge(school: {
|
||||
ofsted_grade?: 1 | 2 | 3 | 4 | null;
|
||||
@@ -686,7 +688,9 @@ export function buildOfstedListBadge(school: {
|
||||
ofsted_date?: string | null;
|
||||
ofsted_framework?: string | null;
|
||||
ofsted_rc_date?: string | null;
|
||||
}): OfstedListBadge {
|
||||
inspectorate?: string | null;
|
||||
ofsted_listed?: boolean | null;
|
||||
}): OfstedListBadge | null {
|
||||
// A report card wins over any carried-forward legacy grade — signalled by
|
||||
// ofsted_rc_date. ofsted_framework is the raw event grouping ("Schools -
|
||||
// S5"), never "ReportCard", so it can't detect report cards.
|
||||
@@ -719,7 +723,9 @@ export function buildOfstedListBadge(school: {
|
||||
return { label: `Inspected${visitYear ? ` · ${visitYear}` : ''}`, cssClass: 'ofstedInspected' };
|
||||
}
|
||||
|
||||
return { label: 'Not yet inspected', cssClass: 'ofstedPending' };
|
||||
if (school.inspectorate === 'ISI') return { label: 'Inspected by ISI', cssClass: 'ofstedPending' };
|
||||
if (school.ofsted_listed) return { label: 'Not yet inspected', cssClass: 'ofstedPending' };
|
||||
return null;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
||||
@@ -146,7 +146,7 @@ with DAG(
|
||||
|
||||
dbt_build_ofsted = BashOperator(
|
||||
task_id="dbt_build",
|
||||
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ stg_ofsted_fe_providers+ stg_ofsted_fe_inspections+ stg_gias_ukprn+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
|
||||
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
|
||||
)
|
||||
|
||||
sync_typesense_ofsted = BashOperator(
|
||||
|
||||
@@ -72,12 +72,6 @@ class GIASEstablishmentsStream(Stream):
|
||||
th.Property("Northing", th.StringType),
|
||||
th.Property("StatutoryLowAge", th.StringType),
|
||||
th.Property("StatutoryHighAge", th.StringType),
|
||||
# Matches Ofsted's further-education providers, which some FE
|
||||
# records name by UKPRN only (audit H4).
|
||||
th.Property("UKPRN", th.StringType),
|
||||
# "ISI" for independent schools the Independent Schools Inspectorate
|
||||
# inspects; "Ofsted" or "Not applicable" otherwise (audit H4).
|
||||
th.Property("InspectorateName (name)", th.StringType),
|
||||
).to_dict()
|
||||
|
||||
def get_records(self, context):
|
||||
|
||||
@@ -1,188 +0,0 @@
|
||||
"""Ofsted's further education and skills management information (audit H4).
|
||||
|
||||
Ofsted publishes FE inspection outcomes on one gov.uk page per academic year
|
||||
("Further education and skills inspections and outcomes: management
|
||||
information from November 2025 to August 2026"). Each page carries "as at"
|
||||
files (one row per provider, with its latest inspection under the previous
|
||||
framework) and cumulative files of renewed-framework inspections, i.e. report
|
||||
cards ("…_10_November_2025_to_31_January_2026.csv", "…_to_28_February_2026.csv",
|
||||
…). Monitoring-visit files are a different series.
|
||||
|
||||
Free of the Singer SDK so CI's pytest, which installs only the backend's
|
||||
requirements, can load it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from datetime import date
|
||||
from typing import Callable, Iterator
|
||||
|
||||
import pandas as pd
|
||||
|
||||
SEARCH_URL = "https://www.gov.uk/api/search.json"
|
||||
SEARCH_QUERY = "further education and skills inspections and outcomes management information"
|
||||
PAGE_TITLE_PREFIX = "Further education and skills inspections and outcomes: management information from "
|
||||
# Renewed-framework (report card) inspections began on 10 November 2025.
|
||||
RENEWED_FRAMEWORK_START = date(2025, 11, 1)
|
||||
|
||||
_MONTHS = {name: number for number, name in enumerate(
|
||||
["january", "february", "march", "april", "may", "june", "july",
|
||||
"august", "september", "october", "november", "december"], start=1)}
|
||||
_TITLE_START = re.compile(r"from ([A-Za-z]+) (\d{4})")
|
||||
_CSV_LINK = re.compile(r'https://assets\.publishing\.service\.gov\.uk/[^"\s<>]+\.csv')
|
||||
_AS_AT = re.compile(r"further_education_and_skills_-_+as_at_(\d{1,2})_([A-Za-z]+)_(\d{4})\.csv$")
|
||||
_PERIOD = re.compile(
|
||||
r"further_education_and_skills_-_+(\d{1,2})_([A-Za-z]+)_(\d{4})_to_(\d{1,2})_([A-Za-z]+)_(\d{4})\.csv$")
|
||||
|
||||
# Ofsted column → Singer field.
|
||||
PROVIDER_COLUMNS = {
|
||||
"Provider URN": "provider_urn",
|
||||
"Provider UKPRN": "provider_ukprn",
|
||||
"Provider name": "provider_name",
|
||||
"Provider type": "provider_type",
|
||||
"Has the provider received a full inspection under the renewed EIF?": "renewed_eif_full_inspection",
|
||||
"First day of inspection of latest inspection before renewed EIF": "graded_inspection_date",
|
||||
"Latest overall effectiveness before renewed EIF": "overall_effectiveness",
|
||||
"Date of latest short inspection before renewed EIF (if relevant)": "short_inspection_date",
|
||||
}
|
||||
INSPECTION_COLUMNS = {
|
||||
"Provider URN": "provider_urn",
|
||||
"Provider UKPRN": "provider_ukprn",
|
||||
"Inspection number": "inspection_number",
|
||||
"First day of inspection": "first_day_of_inspection",
|
||||
"Date published": "date_published",
|
||||
"Safeguarding standards": "safeguarding_standards",
|
||||
}
|
||||
|
||||
|
||||
def _date(day: str, month: str, year: str) -> date | None:
|
||||
number = _MONTHS.get(month.lower())
|
||||
return date(int(year), number, int(day)) if number else None
|
||||
|
||||
|
||||
def _page_start(title: str) -> date | None:
|
||||
match = _TITLE_START.search(title)
|
||||
number = _MONTHS.get(match.group(1).lower()) if match else None
|
||||
return date(int(match.group(2)), number, 1) if number else None
|
||||
|
||||
|
||||
def _dated_pages(results: list[dict]) -> list[tuple[date, str]]:
|
||||
pages = []
|
||||
for result in results:
|
||||
title = result.get("title", "")
|
||||
start = _page_start(title) if title.startswith(PAGE_TITLE_PREFIX) else None
|
||||
if start:
|
||||
pages.append((start, result["link"]))
|
||||
return sorted(pages, reverse=True)
|
||||
|
||||
|
||||
def fe_pages(results: list[dict]) -> list[str]:
|
||||
"""gov.uk paths of the FE MI pages, newest academic year first."""
|
||||
return [link for _, link in _dated_pages(results)]
|
||||
|
||||
|
||||
def renewed_framework_pages(results: list[dict]) -> list[str]:
|
||||
"""The pages that can hold report cards: from November 2025 onwards."""
|
||||
return [link for start, link in _dated_pages(results) if start >= RENEWED_FRAMEWORK_START]
|
||||
|
||||
|
||||
def _latest(html: str, pattern: re.Pattern, date_groups: slice) -> str | None:
|
||||
best = None
|
||||
for link in set(_CSV_LINK.findall(html)):
|
||||
match = pattern.search(link)
|
||||
when = _date(*match.groups()[date_groups]) if match else None
|
||||
if when and (best is None or when > best[0]):
|
||||
best = (when, link)
|
||||
return best[1] if best else None
|
||||
|
||||
|
||||
def latest_as_at_csv(html: str) -> str | None:
|
||||
"""The newest "as at" CSV on a page."""
|
||||
return _latest(html, _AS_AT, slice(0, 3))
|
||||
|
||||
|
||||
def latest_period_csv(html: str) -> str | None:
|
||||
"""The renewed-framework inspections CSV with the latest end date on a page."""
|
||||
return _latest(html, _PERIOD, slice(3, 6))
|
||||
|
||||
|
||||
def providers_csv_url(pages: list[str], fetch: Callable[[str], str]) -> str | None:
|
||||
"""The newest "as at" CSV, from the newest page that has one."""
|
||||
for page in pages:
|
||||
url = latest_as_at_csv(fetch(page))
|
||||
if url:
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def read_mi_csv(content: bytes) -> pd.DataFrame:
|
||||
"""Ofsted's MI CSVs: UTF-8 with a BOM, or Windows-1252. The header is the
|
||||
line naming "Provider URN"."""
|
||||
try:
|
||||
text = content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
text = content.decode("cp1252")
|
||||
lines = text.splitlines()
|
||||
start = next((i for i, line in enumerate(lines[:20]) if "Provider URN" in line), 0)
|
||||
return pd.read_csv(io.StringIO("\n".join(lines[start:])), dtype=str, keep_default_na=False)
|
||||
|
||||
|
||||
def records(df: pd.DataFrame, columns: dict[str, str], key: str) -> Iterator[dict]:
|
||||
"""Singer records: the mapped columns, stripped. Rows without `key` are skipped."""
|
||||
present = {source: field for source, field in columns.items() if source in df.columns}
|
||||
for _, row in df.iterrows():
|
||||
record = {field: str(row[source]).strip() for source, field in present.items()}
|
||||
if record.get(key):
|
||||
yield record
|
||||
|
||||
|
||||
# requests' exceptions are OSErrors; a body that is not JSON is a ValueError.
|
||||
_UNAVAILABLE = (OSError, ValueError)
|
||||
|
||||
|
||||
def provider_records(
|
||||
search: Callable[[], list[dict]],
|
||||
fetch_page: Callable[[str], str],
|
||||
fetch_csv: Callable[[str], bytes],
|
||||
log: Callable[[str], None],
|
||||
) -> Iterator[dict]:
|
||||
"""The newest "as at" file's provider records. gov.uk being unreachable
|
||||
yields nothing, logged: the FE streams share the monthly Ofsted extract
|
||||
with the school MI and must not fail it."""
|
||||
try:
|
||||
url = providers_csv_url(fe_pages(search()), fetch_page)
|
||||
if not url:
|
||||
log("No Ofsted FE 'as at' file found")
|
||||
return
|
||||
df = read_mi_csv(fetch_csv(url))
|
||||
except _UNAVAILABLE as exc:
|
||||
log(f"Ofsted FE providers MI unavailable: {exc}")
|
||||
return
|
||||
yield from records(df, PROVIDER_COLUMNS, "provider_urn")
|
||||
|
||||
|
||||
def inspection_records(
|
||||
search: Callable[[], list[dict]],
|
||||
fetch_page: Callable[[str], str],
|
||||
fetch_csv: Callable[[str], bytes],
|
||||
log: Callable[[str], None],
|
||||
) -> Iterator[dict]:
|
||||
"""Renewed-framework inspection records from every page since November
|
||||
2025. A page that cannot be read is logged and skipped; the others load."""
|
||||
try:
|
||||
pages = renewed_framework_pages(search())
|
||||
except _UNAVAILABLE as exc:
|
||||
log(f"Ofsted FE inspections MI unavailable: {exc}")
|
||||
return
|
||||
for page in pages:
|
||||
try:
|
||||
url = latest_period_csv(fetch_page(page))
|
||||
if not url:
|
||||
log(f"No renewed-framework FE inspections file on {page}")
|
||||
continue
|
||||
df = read_mi_csv(fetch_csv(url))
|
||||
except _UNAVAILABLE as exc:
|
||||
log(f"Ofsted FE inspections MI unavailable for {page}: {exc}")
|
||||
continue
|
||||
yield from records(df, INSPECTION_COLUMNS, "inspection_number")
|
||||
@@ -10,15 +10,6 @@ import requests
|
||||
from singer_sdk import Stream, Tap
|
||||
from singer_sdk import typing as th
|
||||
|
||||
from tap_uk_ofsted.fe_mi import (
|
||||
INSPECTION_COLUMNS,
|
||||
PROVIDER_COLUMNS,
|
||||
SEARCH_QUERY,
|
||||
SEARCH_URL,
|
||||
inspection_records,
|
||||
provider_records,
|
||||
)
|
||||
|
||||
GOV_UK_PAGE = (
|
||||
"https://www.gov.uk/government/statistical-data-sets/"
|
||||
"monthly-management-information-ofsteds-school-inspections-outcomes"
|
||||
@@ -313,62 +304,6 @@ class OfstedInspectionsStream(Stream):
|
||||
self.logger.error("Could not discover Ofsted independent MI download URL")
|
||||
|
||||
|
||||
def _fe_search_results() -> list[dict]:
|
||||
"""gov.uk search results for Ofsted's FE MI pages (one per academic year)."""
|
||||
resp = requests.get(SEARCH_URL, params={"q": SEARCH_QUERY, "count": 50}, timeout=60)
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("results", [])
|
||||
|
||||
|
||||
def _gov_uk_page(path: str) -> str:
|
||||
url = path if path.startswith("http") else "https://www.gov.uk" + path
|
||||
resp = requests.get(url, timeout=60)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
|
||||
def _gov_uk_file(url: str) -> bytes:
|
||||
resp = requests.get(url, timeout=120)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
|
||||
class OfstedFEProvidersStream(Stream):
|
||||
"""Ofsted's further education and skills providers: the newest "as at"
|
||||
file, one row per provider with its latest inspection under the previous
|
||||
framework (audit H4). Page and file selection: fe_mi.py."""
|
||||
|
||||
name = "ofsted_fe_providers"
|
||||
primary_keys = ["provider_urn"]
|
||||
replication_key = None
|
||||
|
||||
schema = th.PropertiesList(
|
||||
th.Property("provider_urn", th.StringType, required=True),
|
||||
*[th.Property(field, th.StringType) for field in PROVIDER_COLUMNS.values() if field != "provider_urn"],
|
||||
).to_dict()
|
||||
|
||||
def get_records(self, context):
|
||||
yield from provider_records(_fe_search_results, _gov_uk_page, _gov_uk_file, self.logger.error)
|
||||
|
||||
|
||||
class OfstedFEInspectionsStream(Stream):
|
||||
"""Ofsted's renewed-framework FE inspections (report cards), from the
|
||||
latest cumulative file on every page since November 2025, so a report
|
||||
card survives the next academic year's page (audit H4)."""
|
||||
|
||||
name = "ofsted_fe_inspections"
|
||||
primary_keys = ["inspection_number"]
|
||||
replication_key = None
|
||||
|
||||
schema = th.PropertiesList(
|
||||
th.Property("inspection_number", th.StringType, required=True),
|
||||
*[th.Property(field, th.StringType) for field in INSPECTION_COLUMNS.values() if field != "inspection_number"],
|
||||
).to_dict()
|
||||
|
||||
def get_records(self, context):
|
||||
yield from inspection_records(_fe_search_results, _gov_uk_page, _gov_uk_file, self.logger.warning)
|
||||
|
||||
|
||||
class TapUKOfsted(Tap):
|
||||
"""Singer tap for UK Ofsted Management Information."""
|
||||
|
||||
@@ -380,7 +315,7 @@ class TapUKOfsted(Tap):
|
||||
).to_dict()
|
||||
|
||||
def discover_streams(self):
|
||||
return [OfstedInspectionsStream(self), OfstedFEProvidersStream(self), OfstedFEInspectionsStream(self)]
|
||||
return [OfstedInspectionsStream(self)]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -24,9 +24,9 @@ DAG_ID = re.compile(r'dag_id="([a-z0-9_]+)"')
|
||||
|
||||
DAILY = 'school_data_daily'
|
||||
|
||||
# dim_school reads int_ofsted_latest and int_ofsted_listed only when the
|
||||
# relations exist (adapter.get_relation), so a missing table is not a failure.
|
||||
OPTIONAL_PARENTS = {'int_ofsted_latest', 'int_ofsted_listed'}
|
||||
# dim_school reads int_ofsted_latest only when the relation exists
|
||||
# (adapter.get_relation), so a missing table is not a failure.
|
||||
OPTIONAL_PARENTS = {'int_ofsted_latest'}
|
||||
|
||||
|
||||
def model_parents():
|
||||
@@ -96,41 +96,3 @@ def test_selected_models_only_read_models_that_exist(dag_id):
|
||||
missing = {model: sorted(PARENTS[model] - available) for model in sorted(selected)
|
||||
if PARENTS[model] - available}
|
||||
assert missing == {}, f'{dag_id} builds models whose parents it never builds: {missing}'
|
||||
|
||||
|
||||
TABLE = re.compile(r"materialized\s*=\s*['\"]table['\"]")
|
||||
|
||||
|
||||
def views():
|
||||
"""Staging and intermediate models are views unless configured as tables."""
|
||||
out = set()
|
||||
for path in MODELS.rglob('*.sql'):
|
||||
layer = path.relative_to(MODELS).parts[0]
|
||||
if layer in ('staging', 'intermediate') and not TABLE.search(path.read_text()):
|
||||
out.add(path.stem)
|
||||
return out
|
||||
|
||||
|
||||
VIEWS = views()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('dag_id', sorted(BUILDS))
|
||||
def test_rebuilt_relations_do_not_drop_views_the_build_reads(dag_id):
|
||||
"""dbt rebuilds a relation by dropping the old one with CASCADE, which drops
|
||||
every view built on it. A view dropped that way and not rebuilt by the same
|
||||
build is gone when that build reads it: the daily GIAS build rebuilt
|
||||
stg_gias_ukprn and dropped the Ofsted views dim_school reads (audit H4)."""
|
||||
selected = selected_models(BUILDS[dag_id], PARENTS)
|
||||
children = defaultdict(set)
|
||||
for model, ps in PARENTS.items():
|
||||
for p in ps:
|
||||
children[p].add(model)
|
||||
dropped, stack = set(), list(selected)
|
||||
while stack:
|
||||
for child in children[stack.pop()]:
|
||||
if child in VIEWS and child not in dropped:
|
||||
dropped.add(child)
|
||||
stack.append(child)
|
||||
dropped -= selected
|
||||
broken = {m: sorted(PARENTS[m] & dropped) for m in sorted(selected) if PARENTS[m] & dropped}
|
||||
assert broken == {}, f'{dag_id} drops views it then reads: {broken}'
|
||||
@@ -1,61 +0,0 @@
|
||||
"""Every field the GIAS establishments stream declares must be a column of the
|
||||
extract GIAS publishes, or it silently loads as null. H4 adds UKPRN (matches
|
||||
Ofsted's further-education providers) and InspectorateName (name) ("ISI" for
|
||||
about 1,340 independent schools). HEADER is GIAS's own, from
|
||||
edubasealldata20261013.csv.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
TAP = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-gias'
|
||||
/ 'tap_uk_gias' / 'tap.py')
|
||||
|
||||
HEADER = (
|
||||
'URN', 'LA (code)', 'LA (name)', 'EstablishmentNumber', 'EstablishmentName',
|
||||
'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)', 'EstablishmentTypeGroup (code)',
|
||||
'EstablishmentTypeGroup (name)', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)',
|
||||
'ReasonEstablishmentOpened (code)', 'ReasonEstablishmentOpened (name)', 'OpenDate',
|
||||
'ReasonEstablishmentClosed (code)', 'ReasonEstablishmentClosed (name)', 'CloseDate',
|
||||
'PhaseOfEducation (code)', 'PhaseOfEducation (name)', 'StatutoryLowAge', 'StatutoryHighAge',
|
||||
'Boarders (code)', 'Boarders (name)', 'NurseryProvision (name)', 'OfficialSixthForm (code)',
|
||||
'OfficialSixthForm (name)', 'Gender (code)', 'Gender (name)', 'ReligiousCharacter (code)',
|
||||
'ReligiousCharacter (name)', 'ReligiousEthos (name)', 'Diocese (code)', 'Diocese (name)',
|
||||
'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)', 'SchoolCapacity', 'SpecialClasses (code)',
|
||||
'SpecialClasses (name)', 'CensusDate', 'NumberOfPupils', 'NumberOfBoys', 'NumberOfGirls',
|
||||
'PercentageFSM', 'TrustSchoolFlag (code)', 'TrustSchoolFlag (name)', 'Trusts (code)',
|
||||
'Trusts (name)', 'SchoolSponsorFlag (name)', 'SchoolSponsors (name)', 'FederationFlag (name)',
|
||||
'Federations (code)', 'Federations (name)', 'UKPRN', 'FEHEIdentifier',
|
||||
'FurtherEducationType (name)', 'LastChangedDate', 'Street', 'Locality', 'Address3', 'Town',
|
||||
'County (name)', 'Postcode', 'SchoolWebsite', 'TelephoneNum', 'HeadTitle (name)',
|
||||
'HeadFirstName', 'HeadLastName', 'HeadPreferredJobTitle', 'BSOInspectorateName (name)',
|
||||
'InspectorateReport', 'DateOfLastInspectionVisit', 'NextInspectionVisit', 'TeenMoth (name)',
|
||||
'TeenMothPlaces', 'CCF (name)', 'SENPRU (name)', 'EBD (name)', 'PlacesPRU', 'FTProv (name)',
|
||||
'EdByOther (name)', 'Section41Approved (name)', 'SEN1 (name)', 'SEN2 (name)', 'SEN3 (name)',
|
||||
'SEN4 (name)', 'SEN5 (name)', 'SEN6 (name)', 'SEN7 (name)', 'SEN8 (name)', 'SEN9 (name)',
|
||||
'SEN10 (name)', 'SEN11 (name)', 'SEN12 (name)', 'SEN13 (name)',
|
||||
'TypeOfResourcedProvision (name)', 'ResourcedProvisionOnRoll', 'ResourcedProvisionCapacity',
|
||||
'SenUnitOnRoll', 'SenUnitCapacity', 'GOR (code)', 'GOR (name)',
|
||||
'DistrictAdministrative (code)', 'DistrictAdministrative (name)', 'AdministrativeWard (code)',
|
||||
'AdministrativeWard (name)', 'ParliamentaryConstituency (code)',
|
||||
'ParliamentaryConstituency (name)', 'UrbanRural (code)', 'UrbanRural (name)',
|
||||
'GSSLACode (name)', 'Easting', 'Northing', 'MSOA (name)', 'LSOA (name)',
|
||||
'InspectorateName (name)', 'SENStat', 'SENNoStat', 'BoardingEstablishment (name)', 'PropsName',
|
||||
'PreviousLA (code)', 'PreviousLA (name)', 'PreviousEstablishmentNumber', 'Country (name)',
|
||||
'UPRN', 'SiteName', 'QABName (code)', 'QABName (name)', 'EstablishmentAccredited (code)',
|
||||
'EstablishmentAccredited (name)', 'QABReport', 'CHNumber', 'MSOA (code)', 'LSOA (code)', 'FSM',
|
||||
'AccreditationExpiryDate',
|
||||
)
|
||||
|
||||
|
||||
def declared():
|
||||
source = TAP.read_text()
|
||||
block = source[source.index('class GIASEstablishmentsStream'):source.index('class GIASLinksStream')]
|
||||
return re.findall(r'th\.Property\("([^"]+)"', block)
|
||||
|
||||
|
||||
def test_every_declared_field_is_a_gias_column():
|
||||
assert set(declared()) - set(HEADER) == set()
|
||||
|
||||
|
||||
def test_the_inspection_fields_are_declared():
|
||||
assert {'UKPRN', 'InspectorateName (name)'} <= set(declared())
|
||||
@@ -1,165 +0,0 @@
|
||||
"""Ofsted publishes further-education inspection outcomes on a gov.uk page per
|
||||
academic year, separate from the school MI the tap already reads (audit H4).
|
||||
The links below are the real ones from the 2025/26 page; the headers are
|
||||
Ofsted's own.
|
||||
"""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-ofsted'
|
||||
/ 'tap_uk_ofsted' / 'fe_mi.py')
|
||||
|
||||
A = 'https://assets.publishing.service.gov.uk/media/'
|
||||
PAGE_2025_26 = f'''
|
||||
<a href="{A}6989b16dc106cc5501b6e4c3/Management_information_-_further_education_and_skills_-_as_at_31_January_2026.csv">x</a>
|
||||
<a href="{A}6989b18cd3f57710b50a9b92/Management_information_-_further_education_and_skills_-_10_November_2025_to_31_January_2026.csv">x</a>
|
||||
<a href="{A}69aec8d6d620c14fa183eeb6/Management_information_-_further_education_and_skills_-__10_November_2025_to_28_February_2026.csv">x</a>
|
||||
<a href="{A}69aec8fcc78869bf8eb8a54a/Management_information_-_further_education_and_skills_-_as_at_28_February_2026.csv">x</a>
|
||||
<a href="{A}69aef091bde9c3f213c89a1e/Management_information_-_further_education_and_skills_-_as_at_31_January_2026.ods">x</a>
|
||||
<a href="{A}6a9fec1e92e72b8ac437eef3/Management_information_-_further_education_and_skills_-_as_at_31_August_2026.csv">x</a>
|
||||
<a href="{A}6a9fed6b5f6e942efe37eed4/Management_information_-_new_provider_monitoring_inspections_-_10_November_2025_to_31_August_2026.csv">x</a>
|
||||
<a href="{A}6a9fed815f6e942efe37eed5/Management_information_-_monitoring_inspections_-_10_November_2025_to_31_August_2026.csv">x</a>
|
||||
<a href="{A}6a9feda65f6e942efe37eed6/Management_information_-_further_education_and_skills_-_10_November_2025_to_31_August_2026.csv">x</a>
|
||||
'''
|
||||
|
||||
SEARCH_RESULTS = [
|
||||
{'title': 'Further education and skills inspections and outcomes: management information from September 2024 to August 2025',
|
||||
'link': '/government/statistical-data-sets/fe-2024-25'},
|
||||
{'title': 'Further education and skills inspections and outcomes: management information from November 2025 to August 2026',
|
||||
'link': '/government/statistical-data-sets/fe-2025-26'},
|
||||
{'title': 'Further education and skills inspections and outcomes: management information from September 2023 to August 2024',
|
||||
'link': '/government/statistical-data-sets/fe-2023-24'},
|
||||
{'title': 'Monthly management information: Ofsted\'s school inspections outcomes',
|
||||
'link': '/government/statistical-data-sets/schools'},
|
||||
]
|
||||
|
||||
PROVIDER_HEADER = (
|
||||
'Provider URN', 'Provider UKPRN', 'Provider name', 'Provider type', 'Provider group',
|
||||
'Local authority', 'Region', 'Ofsted region',
|
||||
'Has the provider received a full inspection under the renewed EIF?',
|
||||
'Has the provider received a new provider monitoring inspection?',
|
||||
'Date of latest short inspection before renewed EIF (if relevant)',
|
||||
'Inspection number of latest full inspection before renewed EIF',
|
||||
'First day of inspection of latest inspection before renewed EIF',
|
||||
'Latest overall effectiveness before renewed EIF',
|
||||
'Latest quality of education before renewed EIF',
|
||||
'Latest behaviour and attitudes before renewed EIF',
|
||||
'Latest personal development before renewed EIF',
|
||||
'Latest leadership and management before renewed EIF',
|
||||
'Latest contribution to meeting skills needs before renewed EIF',
|
||||
'Latest safeguarding before renewed EIF', '',
|
||||
)
|
||||
|
||||
INSPECTION_HEADER = (
|
||||
'Provider URN', 'Provider UKPRN', 'Provider name', 'Provider type', 'Provider group',
|
||||
'Local authority', 'Region', 'Ofsted region', 'Inspection number', 'First day of inspection',
|
||||
'Date published', 'Safeguarding standards', 'Inclusion', 'Leadership and governance',
|
||||
'Contribution to meeting skills needs',
|
||||
'Education programmes for young people - curriculum, teaching and training',
|
||||
'Education programmes for young people - achievement',
|
||||
'Education programmes for young people - participation and development',
|
||||
'Adult learning programmes - curriculum, teaching and training',
|
||||
'Adult learning programmes - achievement',
|
||||
'Adult learning programmes - participation and development',
|
||||
'Apprenticeships - curriculum, teaching and training', 'Apprenticeships - achievement',
|
||||
'Apprenticeships - participation and development',
|
||||
'Provision for learners and apprentices with high needs - curriculum, teaching and training',
|
||||
'Provision for learners and apprentices with high needs - achievement',
|
||||
'Provision for learners and apprentices with high needs - participation and development', '',
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fe():
|
||||
spec = importlib.util.spec_from_file_location('fe_mi', MODULE)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_the_newest_as_at_file_is_chosen(fe):
|
||||
assert fe.latest_as_at_csv(PAGE_2025_26).endswith('_as_at_31_August_2026.csv')
|
||||
|
||||
|
||||
def test_the_latest_cumulative_inspections_file_is_chosen_not_a_monitoring_one(fe):
|
||||
assert fe.latest_period_csv(PAGE_2025_26) == (
|
||||
A + '6a9feda65f6e942efe37eed6/Management_information_-_further_education_and_skills_'
|
||||
'-_10_November_2025_to_31_August_2026.csv')
|
||||
|
||||
|
||||
def test_a_doubled_underscore_file_name_still_parses(fe):
|
||||
page = '\n'.join(line for line in PAGE_2025_26.splitlines() if 'August' not in line)
|
||||
assert fe.latest_period_csv(page).endswith('__10_November_2025_to_28_February_2026.csv')
|
||||
|
||||
|
||||
def test_monitoring_files_alone_give_nothing(fe):
|
||||
page = '\n'.join(line for line in PAGE_2025_26.splitlines() if 'monitoring' in line)
|
||||
assert fe.latest_period_csv(page) is None
|
||||
assert fe.latest_as_at_csv(page) is None
|
||||
|
||||
|
||||
def test_pages_are_the_fe_series_newest_first(fe):
|
||||
assert fe.fe_pages(SEARCH_RESULTS) == [
|
||||
'/government/statistical-data-sets/fe-2025-26',
|
||||
'/government/statistical-data-sets/fe-2024-25',
|
||||
'/government/statistical-data-sets/fe-2023-24',
|
||||
]
|
||||
|
||||
|
||||
def test_report_cards_are_read_only_from_renewed_framework_pages(fe):
|
||||
# Earlier pages hold previous-framework inspection lists, not report cards.
|
||||
assert fe.renewed_framework_pages(SEARCH_RESULTS) == ['/government/statistical-data-sets/fe-2025-26']
|
||||
|
||||
|
||||
def test_providers_fall_back_to_the_previous_page(fe):
|
||||
pages = {'/new': '<p>No files yet</p>', '/old': PAGE_2025_26}
|
||||
assert fe.providers_csv_url(['/new', '/old'], pages.__getitem__).endswith('_as_at_31_August_2026.csv')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('columns, header', [('PROVIDER_COLUMNS', PROVIDER_HEADER),
|
||||
('INSPECTION_COLUMNS', INSPECTION_HEADER)])
|
||||
def test_every_mapped_column_is_in_ofsteds_file(fe, columns, header):
|
||||
assert set(getattr(fe, columns)) - set(header) == set()
|
||||
|
||||
|
||||
def test_csv_reading_and_records(fe):
|
||||
content = ('' + ','.join(f'"{c}"' for c in PROVIDER_HEADER[:14]) + '\n'
|
||||
+ '145175,10005429,Queen Mary\'s College,Sixth form college,Sixth form colleges,Hampshire,'
|
||||
'South East,South East,No,No,-,10227183,06/12/2022,2\n'
|
||||
+ ',,Blank provider,,,,,,,,,,,\n').encode('utf-8')
|
||||
rows = list(fe.records(fe.read_mi_csv(content), fe.PROVIDER_COLUMNS, 'provider_urn'))
|
||||
assert rows == [{
|
||||
'provider_urn': '145175', 'provider_ukprn': '10005429', 'provider_name': "Queen Mary's College",
|
||||
'provider_type': 'Sixth form college', 'renewed_eif_full_inspection': 'No',
|
||||
'graded_inspection_date': '06/12/2022', 'overall_effectiveness': '2', 'short_inspection_date': '-',
|
||||
}]
|
||||
|
||||
|
||||
def test_a_gov_uk_outage_yields_no_providers_rather_than_failing(fe):
|
||||
# The FE streams share the monthly Ofsted extract with the school MI; an
|
||||
# error here must not fail that extract (review of H4).
|
||||
def down():
|
||||
raise ConnectionError('gov.uk unreachable')
|
||||
logged = []
|
||||
assert list(fe.provider_records(down, lambda page: '', lambda url: b'', logged.append)) == []
|
||||
assert logged
|
||||
|
||||
|
||||
def test_one_unreachable_page_does_not_stop_the_other_report_cards(fe):
|
||||
results = [{'title': fe.PAGE_TITLE_PREFIX + 'September 2026 to August 2027', 'link': '/fe-2026-27'},
|
||||
{'title': fe.PAGE_TITLE_PREFIX + 'November 2025 to August 2026', 'link': '/fe-2025-26'}]
|
||||
csv = (','.join(INSPECTION_HEADER[:12]) + '\n'
|
||||
+ '143526,10028480,The Autism Project,,,,,,10427540,12/05/2026,22/07/2026,Met\n').encode()
|
||||
|
||||
def page(path):
|
||||
if path == '/fe-2026-27':
|
||||
raise ConnectionError('timeout')
|
||||
return PAGE_2025_26
|
||||
|
||||
logged = []
|
||||
rows = list(fe.inspection_records(lambda: results, page, lambda url: csv, logged.append))
|
||||
assert [r['inspection_number'] for r in rows] == ['10427540']
|
||||
assert logged
|
||||
@@ -1,98 +0,0 @@
|
||||
-- Intermediate model: Ofsted further education and skills status per URN
|
||||
-- (audit H4), in int_ofsted_latest's columns so it can be unioned there.
|
||||
-- A provider matches a school by Provider URN, else by UKPRN (stg_gias_ukprn).
|
||||
-- Rule, in the C1 status rule's terms
|
||||
-- (docs/superpowers/specs/2026-10-06-inspection-status-beyond-ofsted-schools-design.md):
|
||||
-- a report card on or after the previous-framework inspection wins and
|
||||
-- leaves no grade in force; otherwise a previous-framework grade 1-4 is the
|
||||
-- current grade, dated by that inspection; the latest visit is the later of
|
||||
-- it and any short inspection (whose outcome the MI does not give).
|
||||
|
||||
with gias as (
|
||||
select urn, ukprn from {{ ref('stg_gias_ukprn') }}
|
||||
),
|
||||
|
||||
providers as (
|
||||
select distinct on (g.urn)
|
||||
g.urn,
|
||||
p.graded_inspection_date,
|
||||
p.overall_effectiveness,
|
||||
p.short_inspection_date
|
||||
from gias g
|
||||
join {{ ref('stg_ofsted_fe_providers') }} p
|
||||
on p.provider_urn = g.urn
|
||||
or (g.ukprn is not null and p.provider_ukprn = g.ukprn)
|
||||
order by g.urn, (p.provider_urn = g.urn) desc, p.graded_inspection_date desc nulls last
|
||||
),
|
||||
|
||||
report_cards as (
|
||||
select distinct on (g.urn)
|
||||
g.urn,
|
||||
c.rc_inspection_date,
|
||||
c.rc_safeguarding_met
|
||||
from gias g
|
||||
join {{ ref('stg_ofsted_fe_inspections') }} c
|
||||
on c.provider_urn = g.urn
|
||||
or (g.ukprn is not null and c.provider_ukprn = g.ukprn)
|
||||
order by g.urn, (c.provider_urn = g.urn) desc, c.rc_inspection_date desc, c.inspection_number desc
|
||||
),
|
||||
|
||||
combined as (
|
||||
select
|
||||
coalesce(p.urn, c.urn) as urn,
|
||||
p.graded_inspection_date,
|
||||
p.overall_effectiveness,
|
||||
p.short_inspection_date,
|
||||
c.rc_inspection_date as card_date,
|
||||
c.rc_safeguarding_met as card_safeguarding_met
|
||||
from providers p
|
||||
full outer join report_cards c on c.urn = p.urn
|
||||
),
|
||||
|
||||
decided as (
|
||||
select
|
||||
*,
|
||||
card_date is not null
|
||||
and card_date >= coalesce(greatest(graded_inspection_date, short_inspection_date), card_date)
|
||||
as report_card_latest
|
||||
from combined
|
||||
)
|
||||
|
||||
select
|
||||
urn,
|
||||
case when report_card_latest then card_date
|
||||
else greatest(graded_inspection_date, short_inspection_date) end as latest_visit_date,
|
||||
case when report_card_latest then 'report_card'
|
||||
when short_inspection_date > graded_inspection_date then 'short'
|
||||
else 'graded' end as latest_visit_kind,
|
||||
null::text as latest_visit_outcome,
|
||||
case when not report_card_latest then overall_effectiveness end as current_grade,
|
||||
case when not report_card_latest then graded_inspection_date end as current_grade_date,
|
||||
case when not report_card_latest then 'graded' end as current_grade_basis,
|
||||
graded_inspection_date,
|
||||
null::date as ungraded_inspection_date,
|
||||
coalesce(graded_inspection_date, card_date) as inspection_date,
|
||||
null::text as inspection_type,
|
||||
'Further education and skills'::text as framework,
|
||||
overall_effectiveness,
|
||||
null::integer as quality_of_education,
|
||||
null::integer as behaviour_attitudes,
|
||||
null::integer as personal_development,
|
||||
null::integer as leadership_management,
|
||||
null::integer as early_years_provision,
|
||||
null::integer as sixth_form_provision,
|
||||
null::text as ungraded_outcome,
|
||||
null::integer as ungraded_grade,
|
||||
case when report_card_latest then card_safeguarding_met end as rc_safeguarding_met,
|
||||
null::integer as rc_inclusion,
|
||||
null::integer as rc_curriculum_teaching,
|
||||
null::integer as rc_achievement,
|
||||
null::integer as rc_attendance_behaviour,
|
||||
null::integer as rc_personal_development,
|
||||
null::integer as rc_leadership_governance,
|
||||
null::integer as rc_early_years,
|
||||
null::integer as rc_sixth_form,
|
||||
case when report_card_latest then card_date end as rc_inspection_date,
|
||||
null::text as report_url
|
||||
from decided
|
||||
where report_card_latest or overall_effectiveness between 1 and 4
|
||||
@@ -1,128 +0,0 @@
|
||||
version: 2
|
||||
|
||||
unit_tests:
|
||||
- name: int_ofsted_fe_grade_only
|
||||
description: Queen Mary's College (145175), Good on 6 December 2022, no report card.
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 145175, ukprn: '10005429'}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 145175, provider_ukprn: '10005429', graded_inspection_date: '2022-12-06', overall_effectiveness: 2}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, rc_inspection_date: null, framework: 'Further education and skills'}
|
||||
|
||||
- name: int_ofsted_fe_short_inspection_is_a_visit_not_a_grade
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 1, ukprn: null}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 1, graded_inspection_date: '2019-03-05', overall_effectiveness: 2, short_inspection_date: '2023-02-01'}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 1, latest_visit_date: '2023-02-01', latest_visit_kind: short, current_grade: 2, current_grade_date: '2019-03-05', current_grade_basis: graded}
|
||||
|
||||
- name: int_ofsted_fe_newer_report_card_wins
|
||||
description: The Autism Project - CareTrade (143526). Good 2021; report card 12 May 2026, safeguarding met.
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 143526, ukprn: '10000002'}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 143526, provider_ukprn: '10000002', graded_inspection_date: '2021-11-16', overall_effectiveness: 2}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows:
|
||||
- {inspection_number: '10380001', provider_urn: 143526, provider_ukprn: '10000002', rc_inspection_date: '2026-05-12', rc_safeguarding_met: true}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 143526, latest_visit_date: '2026-05-12', latest_visit_kind: report_card, current_grade: null, current_grade_date: null, current_grade_basis: null, rc_inspection_date: '2026-05-12', rc_safeguarding_met: true, overall_effectiveness: 2}
|
||||
|
||||
- name: int_ofsted_fe_report_card_only
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 150042, ukprn: null}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 150042, graded_inspection_date: null, overall_effectiveness: null}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows:
|
||||
- {inspection_number: '10380002', provider_urn: 150042, rc_inspection_date: '2026-01-20', rc_safeguarding_met: false}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 150042, latest_visit_date: '2026-01-20', latest_visit_kind: report_card, current_grade: null, rc_inspection_date: '2026-01-20', rc_safeguarding_met: false}
|
||||
|
||||
- name: int_ofsted_fe_listed_never_inspected_has_no_row
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 130645, ukprn: null}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 130645, graded_inspection_date: null, overall_effectiveness: null}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows: []
|
||||
expect:
|
||||
rows: []
|
||||
|
||||
- name: int_ofsted_fe_matches_by_ukprn_when_the_urn_differs
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 5, ukprn: '10099999'}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 9999999, provider_ukprn: '10099999', graded_inspection_date: '2024-01-30', overall_effectiveness: 3}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 5, current_grade: 3, current_grade_date: '2024-01-30'}
|
||||
|
||||
- name: int_ofsted_fe_prefers_the_urn_match
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 6, ukprn: '10088888'}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 6, provider_ukprn: '10077777', graded_inspection_date: '2023-05-01', overall_effectiveness: 1}
|
||||
- {provider_urn: 8888888, provider_ukprn: '10088888', graded_inspection_date: '2024-05-01', overall_effectiveness: 4}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 6, current_grade: 1, current_grade_date: '2023-05-01'}
|
||||
|
||||
- name: int_ofsted_fe_report_card_older_than_grade_is_ignored
|
||||
description: Bad data guard. A report card dated before the graded inspection cannot be the latest, and a row must never carry both a grade and a report card.
|
||||
model: int_ofsted_fe_latest
|
||||
given:
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 7, ukprn: null}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 7, graded_inspection_date: '2026-03-01', overall_effectiveness: 2}
|
||||
- input: ref('stg_ofsted_fe_inspections')
|
||||
rows:
|
||||
- {inspection_number: '1', provider_urn: 7, rc_inspection_date: '2026-01-10', rc_safeguarding_met: true}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 7, latest_visit_kind: graded, current_grade: 2, rc_inspection_date: null, rc_safeguarding_met: null}
|
||||
@@ -3,8 +3,6 @@
|
||||
-- inspection) and the overall grade still in force, if any. A grade is dated
|
||||
-- by the inspection that awarded or confirmed it, never by a later visit.
|
||||
-- Rule and examples: docs/superpowers/specs/2026-10-05-ofsted-current-status-design.md
|
||||
-- Schools with no school-framework record take their further education and
|
||||
-- skills status from int_ofsted_fe_latest (same columns, same order; audit H4).
|
||||
|
||||
with inspections as (
|
||||
select
|
||||
@@ -64,64 +62,51 @@ graded as (
|
||||
then 'graded_earlier'
|
||||
end as grade_case
|
||||
from latest
|
||||
),
|
||||
|
||||
school_status as (
|
||||
select
|
||||
urn,
|
||||
latest_visit_date,
|
||||
latest_visit_kind,
|
||||
case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome,
|
||||
case grade_case
|
||||
when 'confirmed' then ungraded_grade
|
||||
when 'graded_latest' then overall_effectiveness
|
||||
when 'graded_earlier' then overall_effectiveness
|
||||
end as current_grade,
|
||||
case grade_case
|
||||
when 'confirmed' then ungraded_inspection_date
|
||||
when 'graded_latest' then graded_inspection_date
|
||||
when 'graded_earlier' then graded_inspection_date
|
||||
end as current_grade_date,
|
||||
case grade_case
|
||||
when 'confirmed' then 'confirmed'
|
||||
when 'graded_latest' then 'graded'
|
||||
when 'graded_earlier' then 'graded'
|
||||
end as current_grade_basis,
|
||||
graded_inspection_date,
|
||||
ungraded_inspection_date,
|
||||
inspection_date,
|
||||
inspection_type,
|
||||
framework,
|
||||
overall_effectiveness,
|
||||
quality_of_education,
|
||||
behaviour_attitudes,
|
||||
personal_development,
|
||||
leadership_management,
|
||||
early_years_provision,
|
||||
sixth_form_provision,
|
||||
ungraded_outcome,
|
||||
ungraded_grade,
|
||||
rc_safeguarding_met,
|
||||
rc_inclusion,
|
||||
rc_curriculum_teaching,
|
||||
rc_achievement,
|
||||
rc_attendance_behaviour,
|
||||
rc_personal_development,
|
||||
rc_leadership_governance,
|
||||
rc_early_years,
|
||||
rc_sixth_form,
|
||||
rc_inspection_date,
|
||||
report_url
|
||||
from graded
|
||||
),
|
||||
|
||||
fe_status as (
|
||||
-- Same columns in the same order as school_status.
|
||||
select f.*
|
||||
from {{ ref('int_ofsted_fe_latest') }} f
|
||||
where not exists (select 1 from school_status s where s.urn = f.urn)
|
||||
)
|
||||
|
||||
select * from school_status
|
||||
union all
|
||||
select * from fe_status
|
||||
select
|
||||
urn,
|
||||
latest_visit_date,
|
||||
latest_visit_kind,
|
||||
case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome,
|
||||
case grade_case
|
||||
when 'confirmed' then ungraded_grade
|
||||
when 'graded_latest' then overall_effectiveness
|
||||
when 'graded_earlier' then overall_effectiveness
|
||||
end as current_grade,
|
||||
case grade_case
|
||||
when 'confirmed' then ungraded_inspection_date
|
||||
when 'graded_latest' then graded_inspection_date
|
||||
when 'graded_earlier' then graded_inspection_date
|
||||
end as current_grade_date,
|
||||
case grade_case
|
||||
when 'confirmed' then 'confirmed'
|
||||
when 'graded_latest' then 'graded'
|
||||
when 'graded_earlier' then 'graded'
|
||||
end as current_grade_basis,
|
||||
graded_inspection_date,
|
||||
ungraded_inspection_date,
|
||||
inspection_date,
|
||||
inspection_type,
|
||||
framework,
|
||||
overall_effectiveness,
|
||||
quality_of_education,
|
||||
behaviour_attitudes,
|
||||
personal_development,
|
||||
leadership_management,
|
||||
early_years_provision,
|
||||
sixth_form_provision,
|
||||
ungraded_outcome,
|
||||
ungraded_grade,
|
||||
rc_safeguarding_met,
|
||||
rc_inclusion,
|
||||
rc_curriculum_teaching,
|
||||
rc_achievement,
|
||||
rc_attendance_behaviour,
|
||||
rc_personal_development,
|
||||
rc_leadership_governance,
|
||||
rc_early_years,
|
||||
rc_sixth_form,
|
||||
rc_inspection_date,
|
||||
report_url
|
||||
from graded
|
||||
@@ -8,8 +8,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 102408, graded_inspection_date: '2025-06-17', ungraded_inspection_date: '2020-02-06', overall_effectiveness: null, ungraded_grade: 2, ungraded_outcome: 'School remains Good'}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 102408, latest_visit_date: '2025-06-17', latest_visit_kind: graded, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null}
|
||||
@@ -20,8 +18,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2019-06-01', current_grade_basis: graded}
|
||||
@@ -33,8 +29,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 104762, graded_inspection_date: '2020-01-07', ungraded_inspection_date: '2024-07-18', overall_effectiveness: 2, ungraded_grade: 2, ungraded_outcome: 'School remains Good'}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 104762, latest_visit_date: '2024-07-18', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: 2, current_grade_date: '2024-07-18', current_grade_basis: confirmed}
|
||||
@@ -46,8 +40,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 139888, graded_inspection_date: '2020-03-03', ungraded_inspection_date: '2025-05-21', overall_effectiveness: 2, ungraded_grade: null, ungraded_outcome: 'Standards maintained'}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 139888, latest_visit_date: '2025-05-21', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: 2, current_grade_date: '2020-03-03', current_grade_basis: graded}
|
||||
@@ -59,8 +51,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 136454, ungraded_inspection_date: '2024-11-13', ungraded_grade: null, ungraded_outcome: 'Standards maintained'}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 136454, latest_visit_date: '2024-11-13', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: null, current_grade_date: null, current_grade_basis: null}
|
||||
@@ -72,8 +62,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 110048, ungraded_inspection_date: '2023-10-05', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2026-05-06', rc_inclusion: 3}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 110048, latest_visit_date: '2026-05-06', latest_visit_kind: report_card, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null}
|
||||
@@ -86,8 +74,6 @@ unit_tests:
|
||||
rows:
|
||||
- {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3}
|
||||
- {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3, rc_inspection_date: '2026-06-02', rc_inclusion: 3}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 138186, latest_visit_date: '2026-06-02', latest_visit_kind: report_card, current_grade: null}
|
||||
@@ -98,8 +84,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 2, graded_inspection_date: '2024-03-01', ungraded_inspection_date: '2024-03-01', overall_effectiveness: 1, ungraded_grade: 2, ungraded_outcome: 'School remains Good'}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 2, latest_visit_kind: graded, current_grade: 1, current_grade_basis: graded}
|
||||
@@ -110,8 +94,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 3, graded_inspection_date: '2018-05-01', overall_effectiveness: 9}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 3, latest_visit_kind: graded, current_grade: null}
|
||||
@@ -126,34 +108,6 @@ unit_tests:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 4, ungraded_inspection_date: '2026-03-02', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2025-12-01', rc_inclusion: 3}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 4, latest_visit_date: '2026-03-02', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: null, current_grade_date: null, current_grade_basis: null}
|
||||
|
||||
- name: fe_status_fills_a_school_with_no_school_record
|
||||
description: Queen Mary's College (145175) has no school-framework record; its FE status passes through (audit H4).
|
||||
model: int_ofsted_latest
|
||||
given:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows: []
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows:
|
||||
- {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, framework: 'Further education and skills'}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, framework: 'Further education and skills'}
|
||||
|
||||
- name: a_school_record_wins_over_an_fe_record
|
||||
model: int_ofsted_latest
|
||||
given:
|
||||
- input: ref('stg_ofsted_inspections')
|
||||
rows:
|
||||
- {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2, framework: 'Schools - S5'}
|
||||
- input: ref('int_ofsted_fe_latest')
|
||||
rows:
|
||||
- {urn: 1, latest_visit_date: '2026-05-12', latest_visit_kind: report_card, rc_inspection_date: '2026-05-12', framework: 'Further education and skills'}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, framework: 'Schools - S5'}
|
||||
@@ -1,21 +0,0 @@
|
||||
-- Intermediate model: every URN Ofsted's own data lists, inspected or not
|
||||
-- (audit H4). Only these may be called "Not yet inspected". Reads the raw
|
||||
-- school MI because stg_ofsted_inspections drops rows with no inspection date.
|
||||
|
||||
with school_mi as (
|
||||
select distinct cast(urn as integer) as urn
|
||||
from {{ source('raw', 'ofsted_inspections') }}
|
||||
where urn is not null
|
||||
),
|
||||
|
||||
fe as (
|
||||
select distinct g.urn
|
||||
from {{ ref('stg_gias_ukprn') }} g
|
||||
join {{ ref('stg_ofsted_fe_providers') }} p
|
||||
on p.provider_urn = g.urn
|
||||
or (g.ukprn is not null and p.provider_ukprn = g.ukprn)
|
||||
)
|
||||
|
||||
select urn from school_mi
|
||||
union
|
||||
select urn from fe
|
||||
@@ -1,29 +0,0 @@
|
||||
version: 2
|
||||
|
||||
unit_tests:
|
||||
- name: int_ofsted_listed_includes_uninspected_schools_and_fe_providers
|
||||
description: >
|
||||
Abbas and Templecombe (150031) is in Ofsted's school MI with no inspection;
|
||||
Exeter and North Devon Colleges Group (130645) is in the FE MI with none;
|
||||
a UKPRN-only FE match counts too. A school in neither is absent (audit H4).
|
||||
model: int_ofsted_listed
|
||||
given:
|
||||
- input: source('raw', 'ofsted_inspections')
|
||||
rows:
|
||||
- {urn: '150031', inspection_date: 'NULL'}
|
||||
- {urn: '102408', inspection_date: '17/06/2025'}
|
||||
- input: ref('stg_gias_ukprn')
|
||||
rows:
|
||||
- {urn: 130645, ukprn: null}
|
||||
- {urn: 5, ukprn: '10099999'}
|
||||
- {urn: 133058, ukprn: null}
|
||||
- input: ref('stg_ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: 130645, provider_ukprn: null}
|
||||
- {provider_urn: 9999999, provider_ukprn: '10099999'}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 150031}
|
||||
- {urn: 102408}
|
||||
- {urn: 130645}
|
||||
- {urn: 5}
|
||||
@@ -138,7 +138,7 @@ models:
|
||||
tests:
|
||||
- not_null
|
||||
- accepted_values:
|
||||
values: ['report_card', 'graded', 'ungraded', 'short']
|
||||
values: ['report_card', 'graded', 'ungraded']
|
||||
- name: current_grade
|
||||
tests:
|
||||
- accepted_values:
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
-- Mart: Canonical school dimension — one row per active URN
|
||||
|
||||
-- int_ofsted_latest and int_ofsted_listed are only ref()'d inside conditional
|
||||
-- blocks below (they may not exist before the Ofsted pipeline first runs), so
|
||||
-- declare the dependencies explicitly for dbt's DAG.
|
||||
-- int_ofsted_latest is only ref()'d inside a conditional block below (it may not
|
||||
-- exist before the Ofsted pipeline first runs), so declare the dependency
|
||||
-- explicitly for dbt's DAG.
|
||||
-- depends_on: {{ ref('int_ofsted_latest') }}
|
||||
-- depends_on: {{ ref('int_ofsted_listed') }}
|
||||
|
||||
with schools as (
|
||||
select * from {{ ref('stg_gias_establishments') }}
|
||||
@@ -16,12 +15,6 @@ with schools as (
|
||||
identifier='int_ofsted_latest'
|
||||
) %}
|
||||
|
||||
{% set listed_relation = adapter.get_relation(
|
||||
database=target.database,
|
||||
schema='intermediate',
|
||||
identifier='int_ofsted_listed'
|
||||
) %}
|
||||
|
||||
select
|
||||
s.urn,
|
||||
s.local_authority_code * 1000 + s.establishment_number as laestab,
|
||||
@@ -77,14 +70,6 @@ select
|
||||
s.status_code,
|
||||
s.nursery_provision,
|
||||
s.admissions_policy_code,
|
||||
-- The inspectorate GIAS names ("ISI" for most independent schools) and
|
||||
-- whether Ofsted's own data lists the school, inspected or not (audit H4).
|
||||
s.inspectorate_name as inspectorate,
|
||||
{% if listed_relation is not none %}
|
||||
l.urn is not null as ofsted_listed,
|
||||
{% else %}
|
||||
null::boolean as ofsted_listed,
|
||||
{% endif %}
|
||||
|
||||
-- Latest Ofsted (populated after monthly Ofsted pipeline runs). The grade
|
||||
-- still in force and the latest visit, from int_ofsted_latest — never a
|
||||
@@ -103,9 +88,6 @@ from schools s
|
||||
{% if ofsted_relation is not none %}
|
||||
left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn
|
||||
{% endif %}
|
||||
{% if listed_relation is not none %}
|
||||
left join {{ ref('int_ofsted_listed') }} l on s.urn = l.urn
|
||||
{% endif %}
|
||||
-- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when
|
||||
-- GIAS flips to Closed — marts fully rebuild each run).
|
||||
where s.status_code in (1, 3)
|
||||
|
||||
@@ -14,8 +14,6 @@ unit_tests:
|
||||
- input: ref('int_ofsted_latest')
|
||||
rows:
|
||||
- {urn: 102408, inspection_date: '2025-06-17', overall_effectiveness: null, ungraded_grade: 2, framework: 'Schools - S5', latest_visit_date: '2025-06-17', current_grade: null}
|
||||
- input: ref('int_ofsted_listed')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 102408, ofsted_grade: null, ofsted_date: '2025-06-17', ofsted_framework: 'Schools - S5'}
|
||||
@@ -32,30 +30,6 @@ unit_tests:
|
||||
- input: ref('int_ofsted_latest')
|
||||
rows:
|
||||
- {urn: 139888, inspection_date: '2020-03-03', overall_effectiveness: 2, ungraded_grade: null, framework: 'Schools - S5', latest_visit_date: '2025-05-21', current_grade: 2}
|
||||
- input: ref('int_ofsted_listed')
|
||||
rows: []
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 139888, ofsted_grade: 2, ofsted_date: '2025-05-21'}
|
||||
|
||||
- name: dim_school_carries_inspectorate_and_ofsted_listing
|
||||
description: >
|
||||
Audit H4. GIAS names ISI for Abbey College Cambridge; Ofsted lists Abbas
|
||||
and Templecombe without an inspection; a third school is in neither.
|
||||
model: dim_school
|
||||
given:
|
||||
- input: ref('stg_gias_establishments')
|
||||
rows:
|
||||
- {urn: 136083, school_name: 'Abbey College Cambridge', status_code: 1, school_type_code: 11, local_authority_code: 873, phase_code: 4, inspectorate_name: 'ISI'}
|
||||
- {urn: 150031, school_name: 'Abbas and Templecombe Church School', status_code: 1, school_type_code: 28, local_authority_code: 933, phase_code: 2, inspectorate_name: 'Not applicable'}
|
||||
- {urn: 1, school_name: 'No data school', status_code: 1, school_type_code: 1, local_authority_code: 312, phase_code: 2, inspectorate_name: null}
|
||||
- input: ref('int_ofsted_latest')
|
||||
rows: []
|
||||
- input: ref('int_ofsted_listed')
|
||||
rows:
|
||||
- {urn: 150031}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 136083, inspectorate: 'ISI', ofsted_listed: false}
|
||||
- {urn: 150031, inspectorate: 'Not applicable', ofsted_listed: true}
|
||||
- {urn: 1, inspectorate: null, ofsted_listed: false}
|
||||
@@ -24,12 +24,6 @@ sources:
|
||||
- name: ofsted_inspections
|
||||
description: Ofsted Management Information inspection records
|
||||
|
||||
- name: ofsted_fe_providers
|
||||
description: Ofsted further education and skills MI, newest "as at" file — one row per provider with its latest inspection under the previous framework
|
||||
|
||||
- name: ofsted_fe_inspections
|
||||
description: Ofsted further education and skills MI — renewed-framework inspections (report cards) since 10 November 2025, one row per inspection
|
||||
|
||||
- name: ees_ks2_attainment
|
||||
description: KS2 school attainment (long format — one row per school × subject × breakdown)
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ renamed as (
|
||||
"UrbanRural (name)" as urban_rural,
|
||||
"ParliamentaryConstituency (name)" as parliamentary_constituency,
|
||||
"NurseryProvision (name)" as nursery_provision,
|
||||
nullif(trim("InspectorateName (name)"), '') as inspectorate_name,
|
||||
cast(nullif("Easting", '') as integer) as easting,
|
||||
cast(nullif("Northing", '') as integer) as northing,
|
||||
cast(nullif("StatutoryLowAge", '') as integer) as statutory_low_age,
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
-- URN → UKPRN from the raw GIAS extract, for matching Ofsted's further
|
||||
-- education providers (int_ofsted_fe_latest, int_ofsted_listed; audit H4).
|
||||
-- A model on the raw source rather than a column of stg_gias_establishments:
|
||||
-- a ref to that model would put the Ofsted models into the daily build's
|
||||
-- stg_gias_establishments+ selection (pipeline/tests/test_dag_selectors.py).
|
||||
-- Built by the monthly Ofsted DAG with the views on it, never by the daily
|
||||
-- one: rebuilding a view drops the views on it (CASCADE), and dim_school, which
|
||||
-- the daily build makes, reads int_ofsted_latest and int_ofsted_listed.
|
||||
|
||||
select
|
||||
cast("URN" as integer) as urn,
|
||||
nullif(trim("UKPRN"), '') as ukprn
|
||||
from {{ source('raw', 'gias_establishments') }}
|
||||
-- target-postgres types URN as bigint (the tap declares an integer).
|
||||
where "URN" is not null
|
||||
@@ -1,29 +0,0 @@
|
||||
version: 2
|
||||
|
||||
unit_tests:
|
||||
- name: stg_ofsted_fe_providers_parses_grades_and_dates
|
||||
description: Ofsted writes "-" for no grade or date; grades are 1-4.
|
||||
model: stg_ofsted_fe_providers
|
||||
given:
|
||||
- input: source('raw', 'ofsted_fe_providers')
|
||||
rows:
|
||||
- {provider_urn: '145175', provider_ukprn: '10005429', provider_name: "Queen Mary's College", renewed_eif_full_inspection: 'No', graded_inspection_date: '06/12/2022', overall_effectiveness: '2', short_inspection_date: '-'}
|
||||
- {provider_urn: '130645', provider_ukprn: '-', provider_name: 'Exeter and North Devon Colleges Group', renewed_eif_full_inspection: 'No', graded_inspection_date: '-', overall_effectiveness: '-', short_inspection_date: '-'}
|
||||
- {provider_urn: '', provider_ukprn: '10000001', provider_name: 'No URN'}
|
||||
expect:
|
||||
rows:
|
||||
- {provider_urn: 145175, provider_ukprn: '10005429', renewed_eif_full_inspection: false, graded_inspection_date: '2022-12-06', overall_effectiveness: 2, short_inspection_date: null}
|
||||
- {provider_urn: 130645, provider_ukprn: null, renewed_eif_full_inspection: false, graded_inspection_date: null, overall_effectiveness: null, short_inspection_date: null}
|
||||
|
||||
- name: stg_ofsted_fe_inspections_parses_dates_and_safeguarding
|
||||
model: stg_ofsted_fe_inspections
|
||||
given:
|
||||
- input: source('raw', 'ofsted_fe_inspections')
|
||||
rows:
|
||||
- {inspection_number: '10380001', provider_urn: '143526', provider_ukprn: '10000002', first_day_of_inspection: '12/05/2026', date_published: '16/06/2026', safeguarding_standards: 'Met'}
|
||||
- {inspection_number: '10380002', provider_urn: '150042', provider_ukprn: '10000003', first_day_of_inspection: '20/01/2026', date_published: '24/02/2026', safeguarding_standards: 'Not met'}
|
||||
- {inspection_number: '10380003', provider_urn: '1', provider_ukprn: '', first_day_of_inspection: '', date_published: '', safeguarding_standards: ''}
|
||||
expect:
|
||||
rows:
|
||||
- {inspection_number: '10380001', provider_urn: 143526, rc_inspection_date: '2026-05-12', published_date: '2026-06-16', rc_safeguarding_met: true}
|
||||
- {inspection_number: '10380002', provider_urn: 150042, rc_inspection_date: '2026-01-20', published_date: '2026-02-24', rc_safeguarding_met: false}
|
||||
@@ -1,18 +0,0 @@
|
||||
-- Staging model: Ofsted's renewed-framework FE inspections (report cards),
|
||||
-- one row per inspection (audit H4). Only the headline is kept: the date and
|
||||
-- the safeguarding result.
|
||||
|
||||
select
|
||||
trim(inspection_number) as inspection_number,
|
||||
cast(trim(provider_urn) as integer) as provider_urn,
|
||||
nullif(nullif(trim(provider_ukprn), ''), '-') as provider_ukprn,
|
||||
to_date(trim(first_day_of_inspection), 'DD/MM/YYYY') as rc_inspection_date,
|
||||
case when trim(date_published) ~ '^\d{2}/\d{2}/\d{4}$'
|
||||
then to_date(trim(date_published), 'DD/MM/YYYY') end as published_date,
|
||||
case lower(trim(safeguarding_standards))
|
||||
when 'met' then true
|
||||
when 'not met' then false
|
||||
end as rc_safeguarding_met
|
||||
from {{ source('raw', 'ofsted_fe_inspections') }}
|
||||
where trim(provider_urn) ~ '^[0-9]+$'
|
||||
and trim(first_day_of_inspection) ~ '^\d{2}/\d{2}/\d{4}$'
|
||||
@@ -1,18 +0,0 @@
|
||||
-- Staging model: Ofsted further education and skills providers — the "as at"
|
||||
-- MI file, one row per provider with its latest inspection under the previous
|
||||
-- framework (audit H4). Ofsted writes '-' for no grade or date.
|
||||
|
||||
select
|
||||
cast(trim(provider_urn) as integer) as provider_urn,
|
||||
nullif(nullif(trim(provider_ukprn), ''), '-') as provider_ukprn,
|
||||
provider_name,
|
||||
provider_type,
|
||||
lower(trim(renewed_eif_full_inspection)) = 'yes' as renewed_eif_full_inspection,
|
||||
case when trim(graded_inspection_date) ~ '^\d{2}/\d{2}/\d{4}$'
|
||||
then to_date(trim(graded_inspection_date), 'DD/MM/YYYY') end as graded_inspection_date,
|
||||
case when trim(overall_effectiveness) in ('1', '2', '3', '4')
|
||||
then trim(overall_effectiveness)::integer end as overall_effectiveness,
|
||||
case when trim(short_inspection_date) ~ '^\d{2}/\d{2}/\d{4}$'
|
||||
then to_date(trim(short_inspection_date), 'DD/MM/YYYY') end as short_inspection_date
|
||||
from {{ source('raw', 'ofsted_fe_providers') }}
|
||||
where trim(provider_urn) ~ '^[0-9]+$'
|
||||
@@ -1,7 +0,0 @@
|
||||
-- A school-framework record always wins over a further education one for the
|
||||
-- same URN (audit H4): no FE row may stand where Ofsted's school MI has one.
|
||||
|
||||
select f.urn
|
||||
from {{ ref('fact_ofsted_latest') }} f
|
||||
join {{ ref('stg_ofsted_inspections') }} s on s.urn = f.urn
|
||||
where f.framework = 'Further education and skills'
|
||||
Reference in new issue
Block a user