Compare commits

..
Author SHA1 Message Date
TudorandClaude Opus 5.5 4f342c50fd docs: H4 rollout order and where stg_gias_ukprn is built (review)
PR Checks / Frontend Typecheck + Tests (pull_request) Waiting to run
PR Checks / Backend Smoke (pull_request) Waiting to run
PR Checks / Build Backend (no push) (pull_request) Waiting to run
PR Checks / Build Frontend (no push) (pull_request) Waiting to run
PR Checks / Build Pipeline (no push) (pull_request) Waiting to run
PR Checks / AI Code Review (Claude) (pull_request) Waiting to run
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:55:49 +01:00
TudorandClaude Opus 5.5 32715af568 fix(ofsted): a gov.uk outage skips the FE streams, not the Ofsted extract (review)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:55:22 +01:00
TudorandClaude Opus 5.5 c16dd0044f fix(dags): build stg_gias_ukprn monthly, never daily (review)
Rebuilding the view in the daily build dropped the Ofsted views on it
(CASCADE), which dim_school reads: every daily run would have failed. A
selector test now rejects any build that drops views it then reads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:54:51 +01:00
TudorandClaude Opus 5.5 cfc9367bb3 fix(dbt): stg_gias_ukprn reads URN as the bigint target-postgres loads (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:28:51 +01:00
TudorandClaude Opus 5.5 8e82e2e4cf feat(dbt): dim_school.inspectorate and ofsted_listed (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:26:42 +01:00
TudorandClaude Opus 5.5 6cc135ec58 feat(dbt): further education status where a school has no school record (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:26:05 +01:00
TudorandClaude Opus 5.5 222886465a feat(dbt): further education status per URN under the C1 rule (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:25:26 +01:00
TudorandClaude Opus 5.5 6abee62c2a feat(dbt): stage Ofsted's further education and skills MI (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:25:04 +01:00
TudorandClaude Opus 5.5 ab3e12e1ac feat(ofsted): extract Ofsted's further education and skills MI (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:24:37 +01:00
TudorandClaude Opus 5.5 1949e85c58 feat(gias): load UKPRN and the inspectorate GIAS names (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:23:31 +01:00
TudorandClaude Opus 5.5 000626c5d6 docs: plan for inspection status beyond Ofsted's school data (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 21:00:54 +01:00
TudorandClaude Opus 5.5 5f7528b399 docs: design for inspection status beyond Ofsted's school data (H4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 23:00:38 +01:00
53 changed files with 3331 additions and 422 deletions

No files matched your search

-8
View File
@@ -1048,11 +1048,6 @@ async def get_school_details(request: Request, urn: int):
"age_range": latest.get("age_range", ""),
"has_sixth_form": latest.get("has_sixth_form"),
"nursery_provision": latest.get("nursery_provision"),
# What the Ofsted badge may say when there is no inspection record
# (audit H4): the inspectorate GIAS names, and whether Ofsted's own
# data lists the school at all.
"inspectorate": latest.get("inspectorate"),
"ofsted_listed": latest.get("ofsted_listed"),
# The header's Selective flag reads it (lib/schoolFacts).
"admissions_policy": latest.get("admissions_policy"),
"status": latest.get("status"),
@@ -1215,9 +1210,6 @@ async def compare_schools(
# schools get entrance-test framing, never the distance template.
"admissions_policy": convert_to_native(latest.get("admissions_policy")),
"has_sixth_form": convert_to_native(latest.get("has_sixth_form")),
# Audit H4: what compare may say when there is no inspection record.
"inspectorate": convert_to_native(latest.get("inspectorate")),
"ofsted_listed": convert_to_native(latest.get("ofsted_listed")),
"capacity": convert_to_native(latest.get("capacity")),
"gias_total_pupils": convert_to_native(latest.get("gias_total_pupils")),
"trust_name": convert_to_native(latest.get("trust_name")),
+1 -8
View File
@@ -251,8 +251,6 @@ _MAIN_QUERY = text("""
s.website,
s.telephone,
s.nursery_provision,
s.inspectorate,
s.ofsted_listed,
foi.current_grade AS ofsted_grade,
foi.current_grade_date AS ofsted_grade_date,
foi.latest_visit_date AS ofsted_date,
@@ -378,8 +376,7 @@ for _old, _new in _LEGACY_NAME_REPLACEMENTS:
_MAIN_QUERY_LEGACY_NAMES = text(_MAIN_QUERY_LEGACY_NAMES)
# Fallback used when marts predate the optional GIAS/location detail columns
# (telephone, nursery provision, county, parliamentary constituency, and the
# inspectorate / Ofsted-listing columns added for audit H4) — i.e. the
# (telephone, nursery provision, county, parliamentary constituency) — i.e. the
# nightly dbt pipeline hasn't rebuilt the mart yet on this DB. Keeps each column
# present as NULL so the whole data load degrades gracefully instead of failing.
_EXTRA_OPTIONAL_COLUMNS = (
@@ -387,8 +384,6 @@ _EXTRA_OPTIONAL_COLUMNS = (
"nursery_provision",
"county",
"parliamentary_constituency",
"inspectorate",
"ofsted_listed",
)
_MAIN_QUERY_NO_EXTRA_COLS = str(_MAIN_QUERY)
for _col, _tbl in (
@@ -396,8 +391,6 @@ for _col, _tbl in (
("nursery_provision", "s"),
("county", "l"),
("parliamentary_constituency", "l"),
("inspectorate", "s"),
("ofsted_listed", "s"),
):
_src = f"{_tbl}.{_col},"
assert _src in _MAIN_QUERY_NO_EXTRA_COLS, (
+3 -6
View File
@@ -39,9 +39,6 @@ def report_card_labels(ofsted: dict) -> dict:
def ofsted_page_url(urn: int) -> str:
"""The school's page on ofsted.gov.uk (all its reports live there — we
never deep-link an individual report). Ofsted's provider pages sit under a
category (21 primary, 23 secondary, 25 special, 40 FE colleges, …); this
address redirects to the right one, where provider/21 404s for every
non-primary school."""
return f"https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/{urn}"
"""The school's page on ofsted.gov.uk (all its reports live there —
we never deep-link an individual report)."""
return f"https://reports.ofsted.gov.uk/provider/21/{urn}"
-2
View File
@@ -578,8 +578,6 @@ SCHOOL_COLUMNS = [
"ofsted_date",
"ofsted_framework",
"ofsted_rc_date",
"inspectorate",
"ofsted_listed",
"latitude",
"longitude",
]
@@ -1,63 +0,0 @@
"""Audit H4: every payload that feeds an Ofsted badge carries the inspectorate
GIAS names and whether Ofsted's own data lists the school, so the site never
says "Not yet inspected" on the strength of missing data."""
import pandas as pd
import pytest
from fastapi.testclient import TestClient
def _df():
base = dict(year=202425, local_authority='Cambridgeshire', local_authority_code=873,
postcode='CB1 1AA', latitude=52.2, longitude=0.12, address='1 High St',
status='Open', town='Cambridge')
return pd.DataFrame([
dict(base, urn=136083, school_name='Abbey College Cambridge', phase='Secondary',
school_type='Other independent school', inspectorate='ISI', ofsted_listed=False),
dict(base, urn=150031, school_name='Abbas and Templecombe Church School', phase='Primary',
school_type='Academy converter', inspectorate='Not applicable', ofsted_listed=True),
])
@pytest.fixture()
def client(monkeypatch):
from backend import app as app_module
from backend import database as database_module
def _no_db():
raise RuntimeError('no database in this test')
monkeypatch.setattr(app_module, 'load_latest_school_data', _df)
monkeypatch.setattr(app_module, 'load_school_data', _df)
monkeypatch.setattr(app_module, 'get_supplementary_data', lambda db, urn: {})
monkeypatch.setattr(app_module, '_place_registry', None)
monkeypatch.setattr(database_module, 'SessionLocal', _no_db)
return TestClient(app_module.app, raise_server_exceptions=False)
def test_the_list_carries_inspectorate_and_listing(client):
resp = client.get('/api/schools?page_size=50')
assert resp.status_code == 200, resp.text
by_urn = {s['urn']: s for s in resp.json()['schools']}
assert by_urn[136083]['inspectorate'] == 'ISI'
assert by_urn[150031]['ofsted_listed'] is True
def test_the_school_page_carries_them(client):
info = client.get('/api/schools/136083').json()['school_info']
assert info['inspectorate'] == 'ISI'
assert info['ofsted_listed'] is False
def test_compare_carries_them(client):
body = client.get('/api/compare?urns=136083,150031').json()
assert body['comparison']['150031']['school_info']['ofsted_listed'] is True
assert body['comparison']['136083']['school_info']['inspectorate'] == 'ISI'
def test_a_mart_built_before_the_columns_still_loads():
# Missing columns make the main query retry with NULLs (data_loader).
from backend import data_loader
assert {'inspectorate', 'ofsted_listed'} <= set(data_loader._EXTRA_OPTIONAL_COLUMNS)
fallback = str(data_loader._MAIN_QUERY_NO_EXTRA_COLS)
assert 'NULL AS inspectorate,' in fallback and 'NULL AS ofsted_listed,' in fallback
+1 -1
View File
@@ -42,4 +42,4 @@ def test_unknown_code_is_skipped_not_crashed():
def test_provider_url():
assert ofsted_page_url(138690) == "https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/138690"
assert ofsted_page_url(138690) == "https://reports.ofsted.gov.uk/provider/21/138690"
@@ -33,7 +33,7 @@ def test_report_card_block_and_provider_url():
assert block["report_card"]["rc_achievement"]["label"] == "Strong standard"
assert "rc_safeguarding_met" not in block["report_card"]
assert block["rc_safeguarding_met"] is True
assert block["ofsted_page_url"] == "https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/100140"
assert block["ofsted_page_url"] == "https://reports.ofsted.gov.uk/provider/21/100140"
def test_no_grade_is_carried_past_a_newer_inspection():
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,256 @@
# Inspection Status Beyond Ofsted's School Data — Design
**Date:** 2026-10-06
**Status:** approved design, not yet implemented
**Scope:** `tap_uk_gias` (two fields), `tap_uk_ofsted` (two FE streams), Ofsted
and GIAS staging, `int_ofsted_latest` / `fact_ofsted_latest`, new
`int_ofsted_fe_latest` and `int_ofsted_listed`, `dim_school`, monthly Ofsted DAG
selector, backend list and detail payloads, search badge, map card, school page
Ofsted section, compare Ofsted rows
**Fixes:** audit finding H4, from the 3 Oct 2026 accuracy audit
## Goal
Never label a school "Not yet inspected" unless Ofsted's own data lists it
without an inspection, and show every inspection result we can source, with its
date.
## The problem
The search badge and the map card (`buildOfstedListBadge`,
`nextjs-app/lib/utils.ts`) print "Not yet inspected" whenever we hold no Ofsted
record. The school page hides its Ofsted section for these schools, and compare
says "No inspection in our dataset". On production on 6 October 2026 (build
`2b5e681`), 2,587 listed schools have no Ofsted record. Joined with GIAS
(6 October 2026) and Ofsted's management information (31 August 2026):
| Group | Schools | Evidence |
|---|---|---|
| Independent schools inspected by ISI | 1,341 | GIAS `InspectorateName` = "ISI". GIAS holds no date, outcome or report link for any of them; ISI publishes no bulk data. |
| Further-education and post-16 providers inspected by Ofsted | 519 | In Ofsted's "Further education and skills" MI: 509 by Provider URN, 10 only by UKPRN through GIAS. 207 FE colleges, 129 special post-16 institutions, 97 higher-education institutions (their FE and apprenticeship provision), 78 16–19 academies and free schools, 8 others. Queen Mary's College (145175): Good, 6 December 2022. |
| Listed by Ofsted, never inspected | 263+ | New academies and free schools present in Ofsted's school MI with every inspection field `NULL`, e.g. Abbas and Templecombe Church School (150031, opened January 2024). "Not yet inspected" is Ofsted's own position for these. |
| No usable inspection data | about 460 | Secure units (inspected as children's homes), higher-education institutions outside the FE MI, newer independents, schools opened after the MI date. |
Of the 519 FE providers: 443 have only a grade from the previous framework
(overall effectiveness 1–4 and the inspection's first day); 45 also have a newer
report card under the renewed framework (from 10 November 2025); 2 have only a
report card; 29 are listed with no inspection. 116 FE providers nationally have
a short inspection after their latest full inspection; the MI gives its date
but not its outcome.
## Non-goals
- Area grades from the previous FE framework, and the report card's
programme-by-theme grid. The FE display is the headline and a link.
- ISI dates or outcomes (no data source).
- A per-school ISI link: ISI's pages use ISI's own identifiers, not URNs.
- Inspections of secure units and children's homes (social care framework).
- Changing how school-framework records are chosen (the C1 rule stands).
## Data
### Extractors
**GIAS (`tap_uk_gias`).** Declare `UKPRN` and `InspectorateName (name)` on the
establishments stream. They are plain values, not codes, so the generated GIAS
dictionaries (backend and pipeline copies, parity-tested) are untouched.
**Ofsted (`tap_uk_ofsted`).** Two streams from Ofsted's "Further education and
skills inspections and outcomes: management information from …" pages on
gov.uk. Pages are found through the gov.uk search API by that title prefix; a
new page appears each academic year.
- `ofsted_fe_providers`: the newest page's newest
`…further_education_and_skills_-_as_at_<date>.csv`. One row per provider:
`Provider URN`, `Provider UKPRN`, `Provider name`, `Provider type`,
`Has the provider received a full inspection under the renewed EIF?`,
`First day of inspection of latest inspection before renewed EIF`,
`Latest overall effectiveness before renewed EIF`,
`Date of latest short inspection before renewed EIF (if relevant)`.
Key: `provider_urn`.
- `ofsted_fe_inspections`: renewed-framework inspections. Each page carries
cumulative files `…further_education_and_skills_-_<start>_to_<end>.csv`
(`10_November_2025_to_31_January_2026`, `…_to_28_February_2026`, …, sometimes
with a doubled underscore); the stream reads the file with the latest end date
from every page since November 2025, so a report card survives the next
year's page. Files named `new_provider_monitoring` or `monitoring_inspections`
are excluded. Fields: `Provider URN`, `Provider UKPRN`, `Inspection number`,
`First day of inspection`, `Date published`, `Safeguarding standards`.
Key: `inspection_number`.
Page and file selection, and the CSV-to-field mapping, live in a module that
imports only pandas and the standard library, so CI's pytest (no
`singer_sdk`) can test them against saved real page HTML and real headers.
The existing school-MI stream is unchanged. It already loads every row Ofsted
lists, inspected or not; `stg_ofsted_inspections` drops the uninspected rows.
### dbt
- `stg_gias_establishments` gains `ukprn` and `inspectorate_name`.
- `stg_ofsted_fe_providers` and `stg_ofsted_fe_inspections`: dates parsed as
`DD/MM/YYYY`; grades `1`–`4` cast to integers and `-` to null; safeguarding
`Met` / `Not met` to a boolean.
- `int_ofsted_fe_latest`, one row per URN we hold. A provider matches a school
by `Provider URN` = URN; failing that, by `Provider UKPRN` = the school's GIAS
`ukprn`. The rule, in the terms of the C1 current-status rule:
| FE record | `current_grade` | `latest_visit` |
|---|---|---|
| Report card newer than the previous-framework inspection (or no such inspection) | null; `rc_inspection_date`, `rc_safeguarding_met` set | the report card (`report_card`) |
| Previous-framework grade 1–4, no newer report card | that grade, dated by the inspection's first day, basis `graded` | the later of that inspection (`graded`) and any short inspection (`short`) |
| Grade `-` and no report card | — | no row |
A short inspection's outcome is not in the data, so it is a visit, never a
grade change.
- `int_ofsted_latest` / `fact_ofsted_latest` take FE rows only for URNs with no
school-framework record; a school record always wins. FE rows carry
`framework = 'Further education and skills'`; school area and report-card
area columns stay null. `latest_visit_kind` gains the value `short`.
- `int_ofsted_listed`: every URN Ofsted's data includes, inspected or not —
all rows of `raw.ofsted_inspections` (before staging's date filter) plus every
matched FE provider.
- `dim_school` gains `inspectorate` (GIAS `inspectorate_name`: "ISI", "Ofsted",
"Not applicable" or null) and `ofsted_listed` (boolean), the latter joined by
the same optional-relation pattern used for `int_ofsted_latest`.
`pipeline/tests/test_dag_selectors.py`'s `OPTIONAL_PARENTS` adds
`int_ofsted_listed`.
- The monthly Ofsted DAG build selects
`stg_ofsted_fe_providers+ stg_ofsted_fe_inspections+ int_ofsted_listed+` as
well.
- Tests: the existing `fact_ofsted_latest` tests (unique URN,
`assert_ofsted_current_grade_consistent`) cover FE rows. A new test asserts no
URN draws on both a school and an FE record.
## What each page shows
**Search badge and map card.** `buildOfstedListBadge` may return no badge. In
order:
1. report card → "Report Card · year";
2. grade in force → grade word · year;
3. inspected, no grade in force → "Inspected · year";
4. GIAS inspectorate "ISI" → "Inspected by ISI" (grey);
5. listed by Ofsted → "Not yet inspected" (grey);
6. otherwise no badge.
FE schools reach 1–3 with the same labels: Queen Mary's College "Good · 2022";
The Autism Project – CareTrade (143526) "Report Card · 2026".
**School page.**
- FE: the existing Ofsted section with "Further education and skills" under the
heading. A grade reads as today ("Good", "Graded inspection, 6 December
2022"); a report card shows its date and the safeguarding result; a later
short inspection gets "Latest visit: Short inspection, [date]". No area or
programme grid. The existing "Ofsted reports ↗" link resolves for FE
providers (Leyton Sixth Form College 130457 → `/provider/40/130457`).
- ISI: an inspection section reading "Inspected by the Independent Schools
Inspectorate (ISI)" with "ISI reports ↗" linking to
`https://www.isi.net/reports/`.
- Listed, never inspected: an Ofsted section reading "Not yet inspected", with
the Ofsted reports link (for a new academy, Ofsted's page links the
predecessor school's reports).
- No data: no section, as today.
**Compare.** FE and listed schools follow the school page's states; ISI schools
read "Inspected by ISI"; schools with no data keep "No inspection in our
dataset".
**Backend.** The list payload and the detail `school_info` gain `inspectorate`
and `ofsted_listed`. FE detail needs no new block: `_ofsted_block` already
carries `current_grade`, `latest_visit`, the report-card date and `framework`.
`inspectorate` and `ofsted_listed` join `_EXTRA_OPTIONAL_COLUMNS` in
`backend/data_loader.py`, so a `dim_school` built before PR 1 makes the main
query retry without them (no badge for groups 4–5) instead of failing.
Copy follows the site's factual rule: inspectorate names and Ofsted's own terms
("Short inspection", "Not yet inspected"), no derived sentences.
## Delivery
Two PRs, as for C1 and C2.
**PR 1 — pipeline:** the GIAS fields, the two FE streams and their selection
module, the staging and intermediate models, `dim_school`'s two columns, the
DAG selector, `OPTIONAL_PARENTS`, unit and data tests.
**PR 2 — site (after the DAGs have run on PR 1):** the backend fields and the
optional-column fallback, the badge rule, the school-page states, compare, and
E2E.
## Testing
- **Extractor (pytest, pandas-only module):** from saved page HTML, the newest
"as at" file and the latest cumulative file per page are chosen and
monitoring files are ignored; the doubled-underscore file name parses; real
headers map to the declared fields.
- **dbt unit tests:** previous-framework grade only; grade with a later short
inspection (grade dated by the full inspection, latest visit `short`); grade
with a newer report card (report card, no grade); report card only;
UKPRN-only match; a school record wins over an FE record; a listed,
uninspected school is in `int_ofsted_listed` and not in `int_ofsted_latest`.
- **Backend pytest:** both new fields in the list and detail payloads; the main
query falls back when either column is missing.
- **Jest:** one case per badge rule including "no badge"; every badge caller
renders nothing when there is no badge; the school page's FE, ISI and
listed-uninspected states; compare cells.
- **E2E:** Queen Mary's College (145175) shows an Ofsted grade, not "Not yet
inspected"; Abbey College Cambridge (136083) shows "Inspected by ISI"; Abbas
and Templecombe Church School (150031) shows "Not yet inspected".
## Rollout and verification
1. Locally, load real GIAS, school MI and FE MI through the changed extractors,
build the models, and check: about 490 FE rows (443 + 45 + 2); 145175 graded
2 on 2022-12-06; 143526 report card 2026-05-12, safeguarding met; 1,341
schools with inspectorate "ISI"; 150031 listed, not in `fact_ofsted_latest`.
2. PR 1 merges; Tudor runs the daily DAG (GIAS fields) and the monthly Ofsted
DAG on staging; the staging API shows the same examples.
3. PR 2 merges, so its E2E journeys run against loaded data.
4. Production, Tudor's decision: promote PR 1, run both DAGs, promote PR 2. If
PR 2 arrives first, the fallback query runs and groups 4–5 show no badge
until the DAGs have run.
5. Validate on production: every "Not yet inspected" belongs to a school
Ofsted's own data lists without an inspection; every "Inspected by ISI"
matches GIAS; every FE grade, date and report card matches Ofsted's FE files;
schools with no data show no badge. Then mark H4 resolved in the audit
report.
## Risks
- **FE page or file naming changes.** Selection is by title prefix and file-name
pattern; if nothing matches, the FE streams emit nothing and FE schools fall
back to "no badge", never a false claim. The extractor tests pin the patterns
against real HTML.
- **A provider's UKPRN shared by several URNs.** Each matched URN takes the
provider's result; the URN match is always tried first.
- **"Not yet inspected" disappears briefly** if PR 2 is promoted before the
DAGs run (fallback query). It never appears wrongly.
## Amendments (implementation plan, 2026-10-13)
- **UKPRN comes from a new view, `stg_gias_ukprn`,** on the raw GIAS source,
not from a `ukprn` column on `stg_gias_establishments`. A `ref` to
`stg_gias_establishments` from the Ofsted models would put them in the daily
build's `stg_gias_establishments+` selection, where their Ofsted parents are
not built (`pipeline/tests/test_dag_selectors.py`). The monthly Ofsted build
selects `stg_gias_ukprn+`; the daily build must not rebuild it, because
rebuilding a view drops the views on it (CASCADE) and `dim_school` reads
them. `stg_gias_establishments` gains only `inspectorate_name`.
- **First rollout order (each environment, once).** The new models need
`raw.gias_establishments."UKPRN"`, which only a daily run with PR 1 creates,
and `dim_school`'s unit tests need `int_ofsted_listed`, which only a monthly
run creates. So: run the daily DAG (its `dbt_build` fails once on
`dim_school`'s unit tests — expected), then `school_data_monthly_ofsted`,
then the daily DAG again. From then on both schedules run normally.
- **A report card is recognised by `rc_inspection_date` as well as by its area
labels** (`hasReportCard`), on the school page and in compare. An FE report
card carries a safeguarding result and no school areas, so the label test
alone read it as "No overall grade".
- **The compare page's Ofsted link is fixed.** `ofsted_page_url` built
`https://reports.ofsted.gov.uk/provider/21/{urn}`; category 21 is primary
schools, so the link 404s for every secondary, special and FE school
(checked: 137086, 101099, 130457, 145175). It now uses
`…/inspection-reports/find-inspection-report/provider/ELS/{urn}`, which
redirects to the right category and is what the school page already uses.
-43
View File
@@ -554,49 +554,6 @@ test('a grade confirmed at an ungraded visit says so and is dated by it', async
await expect(section.getByText('Confirmed at an ungraded inspection, 18 July 2024')).toBeVisible();
});
test('a further education college shows its Ofsted result, not "Not yet inspected"', async ({ page }) => {
// Queen Mary's College (145175) is inspected under Ofsted's further education
// framework, which the site did not load: its badge said "Not yet inspected"
// (audit H4). Its result may change; the label must never be absent or wrong.
const res = await page.request.get('/api/schools/145175');
expect(res.ok()).toBeTruthy();
await page.goto('/school/145175');
const ofsted = page.locator('#ofsted');
await expect(ofsted).toBeVisible();
await expect(ofsted.getByText('Further education and skills')).toBeVisible();
await expect(ofsted).not.toContainText('Not yet inspected');
});
test('an ISI-inspected school says so instead of "Not yet inspected"', async ({ page }) => {
const res = await page.request.get('/api/schools?school_type=independent&phase=secondary&page_size=100');
expect(res.ok()).toBeTruthy();
const school = ((await res.json()).schools ?? []).find(
(s: { inspectorate?: string | null; ofsted_date?: string | null; ofsted_rc_date?: string | null }) =>
s.inspectorate === 'ISI' && !s.ofsted_date && !s.ofsted_rc_date);
test.skip(!school, 'no ISI school without an Ofsted record here');
await page.goto(`/school/${school.urn}`);
await expect(page.getByText('Inspected by the Independent Schools Inspectorate (ISI)')).toBeVisible();
await expect(page.locator('#ofsted')).not.toContainText('Not yet inspected');
});
test('"Not yet inspected" appears only for a school Ofsted lists without an inspection', async ({ page }) => {
const res = await page.request.get('/api/schools?phase=primary&page_size=500');
expect(res.ok()).toBeTruthy();
type Row = { urn: number; inspectorate?: string | null; ofsted_listed?: boolean | null; ofsted_date?: string | null; ofsted_rc_date?: string | null; ofsted_grade?: number | null };
const schools: Row[] = (await res.json()).schools ?? [];
// No Ofsted record, and not an ISI school (whose badge names ISI instead).
const uninspected = (s: Row) => !s.ofsted_date && !s.ofsted_rc_date && !s.ofsted_grade && s.inspectorate !== 'ISI';
const unlisted = schools.find(s => !s.ofsted_listed && uninspected(s));
const listed = schools.find(s => s.ofsted_listed && uninspected(s));
test.skip(!listed, 'no listed, uninspected primary on this page');
await page.goto(`/school/${listed!.urn}`);
await expect(page.locator('#ofsted').getByText('Not yet inspected')).toBeVisible();
if (unlisted) {
await page.goto(`/school/${unlisted.urn}`);
await expect(page.getByText('Not yet inspected')).toHaveCount(0);
}
});
test('school detail page renders name and performance data', async ({ page }) => {
await searchByName(page, 'primary');
const firstSchool = schoolLinks(page).first();
@@ -69,18 +69,6 @@ const data: Record<string, ComparisonData> = {
};
describe('CompareOfsted', () => {
it('words a school without an Ofsted record from the evidence (audit H4)', () => {
const isi = { ...school(4, 'ISI School'), inspectorate: 'ISI' } as School;
const listed = { ...school(5, 'New Academy'), ofsted_listed: true } as School;
render(<CompareOfsted schools={[isi, listed]} data={{
'4': { school_info: isi, yearly_data: [] } as unknown as ComparisonData,
'5': { school_info: listed, yearly_data: [] } as unknown as ComparisonData,
}} />);
expect(screen.getAllByText('Inspected by ISI').length).toBeGreaterThan(0);
expect(screen.getAllByText('Not yet inspected').length).toBeGreaterThan(0);
expect(screen.queryByText(/No inspection outcome in our dataset/)).toBeNull();
});
it('renders the three regimes without inventing an overall grade for report cards', () => {
render(<CompareOfsted schools={schools} data={data} />);
@@ -1,20 +0,0 @@
import { render, screen } from '@testing-library/react';
import { InspectionNotice } from '@/components/school/InspectionNotice';
it('names ISI and links its reports for an ISI school (audit H4)', () => {
render(<InspectionNotice school={{ urn: 136083, inspectorate: 'ISI', ofsted_listed: false }} />);
expect(screen.getByText('Inspected by the Independent Schools Inspectorate (ISI)')).toBeInTheDocument();
expect(screen.getByRole('link', { name: /ISI reports/ })).toHaveAttribute('href', 'https://www.isi.net/reports/');
});
it('says "Not yet inspected" for a school Ofsted lists without an inspection', () => {
render(<InspectionNotice school={{ urn: 150031, inspectorate: 'Not applicable', ofsted_listed: true }} />);
expect(screen.getByText('Not yet inspected')).toBeInTheDocument();
expect(screen.getByRole('link', { name: /Ofsted reports/ })).toHaveAttribute(
'href', 'https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/150031');
});
it('renders nothing without inspection data, including an unknown listing', () => {
const { container } = render(<InspectionNotice school={{ urn: 1, inspectorate: null, ofsted_listed: null }} />);
expect(container).toBeEmptyDOMElement();
});
@@ -82,14 +82,3 @@ it('prints an ungraded outcome when there is no grade (Oakgrove)', () => {
expect(screen.getByText('No overall grade')).toBeInTheDocument();
expect(screen.getByText(/Ungraded inspection, 13 November 2024: Standards maintained/)).toBeInTheDocument();
});
it('labels a further education record and prints its short inspection (audit H4)', () => {
renderSection({
framework: 'Further education and skills',
current_grade: { grade: 2, date: '2019-03-05', basis: 'graded' },
latest_visit: { date: '2023-02-01', kind: 'short', outcome: null },
});
expect(screen.getByText('Further education and skills')).toBeInTheDocument();
expect(screen.getByText('Good')).toBeInTheDocument();
expect(screen.getByText('Latest visit: Short inspection, 1 February 2023')).toBeInTheDocument();
});
@@ -52,8 +52,3 @@ describe('SchoolRow shares the school page flags', () => {
expect(screen.getByText("Boys' school")).toBeInTheDocument();
});
});
it('renders no Ofsted badge for a school with no inspection data (audit H4)', () => {
const { container } = render(<SchoolRow school={{ ...base, ofsted_grade: null, ofsted_date: null, ofsted_listed: false, inspectorate: null }} />);
expect(container.textContent).not.toMatch(/Not yet inspected|Inspected/);
});
@@ -19,7 +19,6 @@ import {
summariseAdmissions,
summariseReportCard,
verdict,
noInspectionLabel,
} from '@/lib/compareLogic';
import type { OfstedInspection, SchoolAdmissions } from '@/lib/types';
@@ -360,14 +359,3 @@ describe('basket limits per comparison group', () => {
expect(kept).toEqual([0, 1, 2, 3, 4, 7, 8, 9]);
});
});
it('shows an FE report card with only a safeguarding result as a report card (audit H4)', () => {
const display = ofstedDisplay(ofsted({ rc_inspection_date: '2026-05-12', rc_safeguarding_met: true, report_card: {} }));
expect(display.kind).toBe('report_card');
});
it('words the no-record compare cell from the evidence', () => {
expect(noInspectionLabel({ inspectorate: 'ISI' })).toBe('Inspected by ISI');
expect(noInspectionLabel({ ofsted_listed: true })).toBe('Not yet inspected');
expect(noInspectionLabel({ ofsted_listed: null })).toBe('No inspection in our dataset');
});
+1 -10
View File
@@ -1,4 +1,4 @@
import { gradeSourceLine, hasReportCard, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
import { gradeSourceLine, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
import type { OfstedInspection } from '@/lib/types';
const base = { report_card: {} } as unknown as OfstedInspection;
@@ -45,12 +45,3 @@ describe('showLatestVisitLine', () => {
latest_visit: { date: '2025-06-17', kind: 'graded', outcome: null } })).toBe(false);
});
});
it('names a short inspection as Ofsted does', () => {
expect(latestVisitLine({ date: '2023-02-01', kind: 'short', outcome: null })).toBe('Short inspection, 1 February 2023');
});
it('treats a report card with only a safeguarding result as a report card (FE)', () => {
expect(hasReportCard({ rc_inspection_date: '2026-05-12', report_card: {} })).toBe(true);
expect(hasReportCard({ rc_inspection_date: null, report_card: {} })).toBe(false);
});
+26 -38
View File
@@ -148,40 +148,40 @@ describe('ofstedLegacyAreas', () => {
describe('buildOfstedListBadge', () => {
it('returns grade word + year for OEIF Outstanding', () => {
const badge = buildOfstedListBadge({ ofsted_grade: 1, ofsted_grade_date: '2023-11-15', ofsted_date: '2023-11-15', ofsted_framework: 'OEIF' });
expect(badge!.label).toBe('Outstanding · 2023');
expect(badge!.cssClass).toBe('ofsted1');
expect(badge.label).toBe('Outstanding · 2023');
expect(badge.cssClass).toBe('ofsted1');
});
it('returns grade word for each OEIF grade', () => {
expect(buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2022-05-01' })!.label).toBe('Good · 2022');
expect(buildOfstedListBadge({ ofsted_grade: 3, ofsted_grade_date: '2021-01-01' })!.label).toBe('Req. Improvement · 2021');
expect(buildOfstedListBadge({ ofsted_grade: 4, ofsted_grade_date: '2020-03-01' })!.label).toBe('Inadequate · 2020');
expect(buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2022-05-01' }).label).toBe('Good · 2022');
expect(buildOfstedListBadge({ ofsted_grade: 3, ofsted_grade_date: '2021-01-01' }).label).toBe('Req. Improvement · 2021');
expect(buildOfstedListBadge({ ofsted_grade: 4, ofsted_grade_date: '2020-03-01' }).label).toBe('Inadequate · 2020');
});
it('prints a grade without a year when its date is missing', () => {
const badge = buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: null, ofsted_date: '2025-01-01' });
expect(badge!.label).toBe('Good');
expect(badge!.cssClass).toBe('ofsted2');
expect(badge.label).toBe('Good');
expect(badge.cssClass).toBe('ofsted2');
});
it('dates a grade by the inspection that awarded it, not the latest visit', () => {
// Washwood Heath: Good from a March 2020 graded inspection; latest visit
// an ungraded one in May 2025 (audit M1).
const badge = buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2020-03-03', ofsted_date: '2025-05-21' });
expect(badge!.label).toBe('Good · 2020');
expect(badge.label).toBe('Good · 2020');
});
it('shows Inspected for a latest inspection that gave no grade (Rabbsfarm)', () => {
// Audit C1: this used to read "Good · 2025", from a 2020 ungraded visit.
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_grade_date: null, ofsted_date: '2025-06-17' });
expect(badge!.label).toBe('Inspected · 2025');
expect(badge!.cssClass).toBe('ofstedInspected');
expect(badge.label).toBe('Inspected · 2025');
expect(badge.cssClass).toBe('ofstedInspected');
});
it('returns a Report Card badge when ofsted_rc_date is present', () => {
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_rc_date: '2026-02-03' });
expect(badge!.label).toBe('Report Card · 2026');
expect(badge!.cssClass).toBe('ofstedRc');
expect(badge.label).toBe('Report Card · 2026');
expect(badge.cssClass).toBe('ofstedRc');
});
it('a report card wins over a carried-forward legacy grade', () => {
@@ -194,46 +194,34 @@ describe('buildOfstedListBadge', () => {
ofsted_framework: 'Schools - S5',
ofsted_rc_date: '2026-02-03',
});
expect(badge!.label).toBe('Report Card · 2026');
expect(badge!.cssClass).toBe('ofstedRc');
expect(badge.label).toBe('Report Card · 2026');
expect(badge.cssClass).toBe('ofstedRc');
});
it('returns an "Inspected" badge for an OEIF inspection with no overall grade (post-Sept-2024)', () => {
// Inspected after Sept 2024: inspection on record (date + framework) but
// Ofsted no longer issues an overall grade. Must NOT read as "Not yet inspected".
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: '2024-11-01', ofsted_framework: 'OEIF' });
expect(badge!.label).toBe('Inspected · 2024');
expect(badge!.cssClass).toBe('ofstedInspected');
expect(badge.label).toBe('Inspected · 2024');
expect(badge.cssClass).toBe('ofstedInspected');
});
it('returns an "Inspected" badge without a year when the grade is missing and date is absent but a record exists', () => {
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_framework: 'OEIF' });
expect(badge!.label).toBe('Inspected');
expect(badge!.cssClass).toBe('ofstedInspected');
expect(badge.label).toBe('Inspected');
expect(badge.cssClass).toBe('ofstedInspected');
});
it('says "Not yet inspected" only when Ofsted lists the school uninspected (audit H4)', () => {
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_listed: true });
expect(badge).toEqual({ label: 'Not yet inspected', cssClass: 'ofstedPending' });
it('returns pending badge when no grade and no inspection on record', () => {
const badge = buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_framework: null });
expect(badge.label).toBe('Not yet inspected');
expect(badge.cssClass).toBe('ofstedPending');
});
it('names ISI for a school GIAS says ISI inspects', () => {
expect(buildOfstedListBadge({ ofsted_date: null, inspectorate: 'ISI', ofsted_listed: false }))
.toEqual({ label: 'Inspected by ISI', cssClass: 'ofstedPending' });
});
it('an Ofsted record wins over ISI', () => {
expect(buildOfstedListBadge({ ofsted_grade: 2, ofsted_grade_date: '2022-03-01', inspectorate: 'ISI' })!.label)
.toBe('Good · 2022');
});
it('gives no badge without any inspection data', () => {
expect(buildOfstedListBadge({ ofsted_grade: null, ofsted_date: null, ofsted_framework: null })).toBeNull();
expect(buildOfstedListBadge({ ofsted_listed: null, inspectorate: 'Not applicable' })).toBeNull();
});
it('gives no badge when all fields are undefined (audit H4)', () => {
expect(buildOfstedListBadge({})).toBeNull();
it('returns pending badge when all fields are undefined', () => {
const badge = buildOfstedListBadge({});
expect(badge.label).toBe('Not yet inspected');
expect(badge.cssClass).toBe('ofstedPending');
});
});
+1 -1
View File
@@ -104,7 +104,7 @@ function cardHtml(school: School, ctx: CardContext): string {
? `${metric ? ' · ' : ''}${school.total_pupils.toLocaleString('en-GB')} pupils` : '';
return `<div class="sc-card">
<div class="sc-card-head"><a class="sc-card-name" href="${href}">${escapeHtml(school.school_name)}</a>${distance}</div>
<div class="sc-card-tags">${badge ? `<span class="sc-ofsted sc-${badge.cssClass}">${escapeHtml(badge.label)}</span>` : ''}${type}</div>
<div class="sc-card-tags"><span class="sc-ofsted sc-${badge.cssClass}">${escapeHtml(badge.label)}</span>${type}</div>
${metric || pupils ? `<div class="sc-card-metric">${metric}${pupils}</div>` : ''}
<div class="sc-card-actions">
<a class="btn btn-tertiary btn-sm" href="${href}">View</a>
+3 -5
View File
@@ -66,11 +66,9 @@ export function SchoolRow({
<a href={schoolUrl(school.urn, school.school_name)} className={styles.schoolName}>
{school.school_name}
</a>
{ofstedBadge && (
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
{ofstedBadge.label}
</span>
)}
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
{ofstedBadge.label}
</span>
</div>
{/* Line 2: Context tags */}
+3 -5
View File
@@ -67,11 +67,9 @@ export function SecondarySchoolRow({
<a href={schoolUrl(school.urn, school.school_name)} className={styles.schoolName}>
{school.school_name}
</a>
{ofstedBadge && (
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
{ofstedBadge.label}
</span>
)}
<span className={`${styles.ofstedBadge} ${styles[ofstedBadge.cssClass]}`}>
{ofstedBadge.label}
</span>
</div>
{/* Line 2: Context tags */}
@@ -14,7 +14,6 @@ import {
summariseAdmissions,
verdict,
type ReportCardSummary,
noInspectionLabel,
} from '@/lib/compareLogic';
import type { Benchmarks, ComparisonData, NationalAverages, School } from '@/lib/types';
import { Cell, Chip, Measure, Section, SectionGrid, sectionStyles as s } from './sectionShared';
@@ -92,7 +91,7 @@ export function CompareAtAGlance({
No overall grade
</span>
)}
{display.kind === 'none' && <span className={s.small}>{noInspectionLabel(data[String(school.urn)]?.school_info ?? school)}</span>}
{display.kind === 'none' && <span className={s.small}>No inspection in our dataset</span>}
</Cell>
);
})}
@@ -12,7 +12,6 @@ import {
ofstedDisplay,
rcAreaLabel,
type OfstedDisplay,
noInspectionLabel,
} from '@/lib/compareLogic';
import { gradeSourceLine } from '@/lib/ofstedStatus';
import type { ComparisonData, OfstedCurrentGrade, OfstedInspection, School } from '@/lib/types';
@@ -42,11 +41,9 @@ function yearsSince(iso: string | null): number | null {
return (Date.now() - d.getTime()) / (365.25 * 24 * 3600 * 1000);
}
function ResultCell({ display, current, school }: {
display: OfstedDisplay; current: OfstedCurrentGrade | null; school: School;
}) {
function ResultCell({ display, current }: { display: OfstedDisplay; current: OfstedCurrentGrade | null }) {
if (display.kind === 'none') {
return <span className={s.small}>{noInspectionLabel(school)}</span>;
return <span className={s.small}>No inspection outcome in our dataset</span>;
}
if (display.kind === 'report_card') {
return (
@@ -178,7 +175,7 @@ export function CompareOfsted({
<Measure label="Result">
{schools.map((school, i) => (
<Cell key={school.urn} school={school} index={i}>
<ResultCell display={displays[i]} current={data[String(school.urn)]?.ofsted?.current_grade ?? null} school={data[String(school.urn)]?.school_info ?? school} />
<ResultCell display={displays[i]} current={data[String(school.urn)]?.ofsted?.current_grade ?? null} />
</Cell>
))}
</Measure>
@@ -216,7 +213,7 @@ export function CompareOfsted({
schoolName={school.school_name}
/>
) : (
<span className={s.small}>{noInspectionLabel(data[String(school.urn)]?.school_info ?? school)}</span>
<span className={s.small}>No inspection in our dataset</span>
)}
</Cell>
);
@@ -231,7 +228,7 @@ export function CompareOfsted({
{schools.map((school, i) => {
const url =
data[String(school.urn)]?.ofsted?.ofsted_page_url ??
`https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/${school.urn}`;
`https://reports.ofsted.gov.uk/provider/21/${school.urn}`;
return (
<Cell key={school.urn} school={school} index={i}>
{/* Short visible label to save space on mobile (the coloured
@@ -1,46 +0,0 @@
/**
* InspectionNotice — the school page's inspection section when there is no
* Ofsted inspection record (audit H4): the inspectorate GIAS names (ISI), or
* Ofsted's own "not yet inspected" when its data lists the school. Nothing
* otherwise: no claim the data cannot support. Server component.
*/
import type { School } from '@/lib/types';
import { Section, sectionStyles as styles } from './sectionShared';
const OFSTED_REPORTS = 'https://reports.ofsted.gov.uk/inspection-reports/find-inspection-report/provider/ELS/';
const ISI_REPORTS = 'https://www.isi.net/reports/';
export function InspectionNotice({ school }: { school: Pick<School, 'urn' | 'inspectorate' | 'ofsted_listed'> }) {
if (school.inspectorate === 'ISI') {
return (
<Section id="ofsted">
<h2 className={styles.sectionTitle}>
Inspection
<a href={ISI_REPORTS} target="_blank" rel="noopener noreferrer" className={styles.ofstedReportLink}
data-umami-event="external_link_clicked" data-umami-event-target="isi">
ISI reports ↗
</a>
</h2>
<p className={styles.ofstedDisclaimer}>Inspected by the Independent Schools Inspectorate (ISI)</p>
</Section>
);
}
if (school.ofsted_listed) {
return (
<Section id="ofsted">
<h2 className={styles.sectionTitle}>
Ofsted Rating
<a href={`${OFSTED_REPORTS}${school.urn}`} target="_blank" rel="noopener noreferrer"
className={styles.ofstedReportLink} data-umami-event="external_link_clicked" data-umami-event-target="ofsted">
Ofsted reports ↗
</a>
</h2>
<div className={styles.ofstedHeader}>
<span className={styles.ofstedGrade}>Not yet inspected</span>
</div>
</Section>
);
}
return null;
}
@@ -12,7 +12,7 @@
*/
import { ofstedDisplay } from '@/lib/compareLogic';
import { FE_FRAMEWORK, formatOfstedDate, gradeSourceLine, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
import { formatOfstedDate, gradeSourceLine, latestVisitLine, showLatestVisitLine } from '@/lib/ofstedStatus';
import type { OfstedInspection } from '@/lib/types';
import { Section, sectionStyles as styles } from './sectionShared';
@@ -71,10 +71,6 @@ export function OfstedSection({
</a>
</h2>
{ofsted.framework === FE_FRAMEWORK && (
<p className={styles.ofstedDisclaimer}>{FE_FRAMEWORK}</p>
)}
{isReportCard ? (
/* ── New Report Card layout ── */
<>
@@ -19,8 +19,6 @@ import type {
import { ofstedLegacyAreas } from '@/lib/utils';
import type { SchoolFlags } from '@/lib/schoolSections';
import { OfstedSection } from './OfstedSection';
import { InspectionNotice } from './InspectionNotice';
import { hasReportCard } from '@/lib/ofstedStatus';
import { ResultsSection } from './ResultsSection';
import { AdmissionsSection } from './AdmissionsSection';
import { DistanceSection } from './DistanceSection';
@@ -56,8 +54,7 @@ export function PrimarySchoolSections({
const primaryAvg = nationalAvg?.primary ?? {};
const secondaryAvg = nationalAvg?.secondary ?? {};
// By date as well as area labels: an FE report card has only a safeguarding result (audit H4).
const isReportCard = !!ofsted && hasReportCard(ofsted);
const isReportCard = !!(ofsted?.report_card && Object.keys(ofsted.report_card).length > 0);
// Report cards are dated by their own inspection (rc_inspection_date);
// anything else by the school's latest visit, never the older inspection a
// grade may come from.
@@ -74,7 +71,7 @@ export function PrimarySchoolSections({
return (
<>
{ofsted ? (
{ofsted && (
<OfstedSection
ofsted={ofsted}
urn={schoolInfo.urn}
@@ -83,7 +80,7 @@ export function PrimarySchoolSections({
oeifAllSameGrade={oeifAllSameGrade}
oeifAreas={oeifAreas}
/>
) : <InspectionNotice school={schoolInfo} />}
)}
{flags.hasAnyResults && flags.latestResults && (
<ResultsSection
@@ -19,8 +19,6 @@ import type {
import { ofstedLegacyAreas } from '@/lib/utils';
import type { SecondaryFlags } from '@/lib/schoolSections';
import { OfstedSection } from './OfstedSection';
import { InspectionNotice } from './InspectionNotice';
import { hasReportCard } from '@/lib/ofstedStatus';
import { GcseSection } from './GcseSection';
import { DestinationsSection } from './DestinationsSection';
import { Post16DestinationsSection } from './Post16DestinationsSection';
@@ -62,8 +60,7 @@ export function SecondarySchoolSections({
}: SecondarySchoolSectionsProps) {
const secondaryAvg = nationalAvg?.secondary ?? {};
// By date as well as area labels: an FE report card has only a safeguarding result (audit H4).
const isReportCard = !!ofsted && hasReportCard(ofsted);
const isReportCard = !!(ofsted?.report_card && Object.keys(ofsted.report_card).length > 0);
const ofstedInspectedDate = isReportCard
? ofsted?.rc_inspection_date ?? null
: ofsted?.latest_visit?.date ?? null;
@@ -77,7 +74,7 @@ export function SecondarySchoolSections({
return (
<div className={styles.secondaryScope}>
{ofsted ? (
{ofsted && (
<OfstedSection
ofsted={ofsted}
urn={schoolInfo.urn}
@@ -86,7 +83,7 @@ export function SecondarySchoolSections({
oeifAllSameGrade={oeifAllSameGrade}
oeifAreas={oeifAreas}
/>
) : <InspectionNotice school={schoolInfo} />}
)}
{flags.hasResults && flags.latestResults && (
<GcseSection
+2 -13
View File
@@ -10,8 +10,7 @@
* methodology instead of thresholding point estimates.
*/
import { hasReportCard } from './ofstedStatus';
import type { OfstedInspection, School, SchoolAdmissions } from './types';
import type { OfstedInspection, SchoolAdmissions } from './types';
// ---------------------------------------------------------------------------
// Verdicts against an anchor (England average or state-school benchmark)
@@ -105,16 +104,6 @@ export type OfstedDisplay =
| { kind: 'no_overall_grade' }
| { kind: 'report_card'; summary: ReportCardSummary };
/**
* The compare cell when a school has no Ofsted record (audit H4): ISI per GIAS,
* Ofsted's own "not yet inspected", or plainly that we hold nothing.
*/
export function noInspectionLabel(school: Pick<School, 'inspectorate' | 'ofsted_listed'>): string {
if (school.inspectorate === 'ISI') return 'Inspected by ISI';
if (school.ofsted_listed) return 'Not yet inspected';
return 'No inspection in our dataset';
}
export function ofstedDisplay(
ofsted: OfstedInspection | null | undefined,
): OfstedDisplay {
@@ -122,7 +111,7 @@ export function ofstedDisplay(
// A report card is the newest inspection format; when present it wins —
// never derive or prefer an overall grade alongside it.
if (hasReportCard(ofsted)) {
if (ofsted.report_card && Object.keys(ofsted.report_card).length > 0) {
return { kind: 'report_card', summary: summariseReportCard(ofsted) };
}
-14
View File
@@ -17,22 +17,8 @@ const VISIT_KIND: Record<OfstedLatestVisit['kind'], string> = {
report_card: 'Report card inspection',
graded: 'Graded inspection',
ungraded: 'Ungraded inspection',
short: 'Short inspection',
};
/** fact_ofsted_latest.framework for Ofsted's further education and skills records (audit H4). */
export const FE_FRAMEWORK = 'Further education and skills';
/**
* A report card is in force. Checked by its date as well as its area labels:
* an FE report card carries a safeguarding result and no school areas.
*/
export function hasReportCard(
ofsted: Pick<OfstedInspection, 'rc_inspection_date' | 'report_card'>,
): boolean {
return !!ofsted.rc_inspection_date || Object.keys(ofsted.report_card ?? {}).length > 0;
}
/** "Graded inspection, 6 July 2016" or "Confirmed at an ungraded inspection, 14 March 2023". */
export function gradeSourceLine(current: OfstedCurrentGrade): string {
const when = formatOfstedDate(current.date);
+2 -7
View File
@@ -86,10 +86,6 @@ export interface School {
/** The school's latest inspection of any kind. */
ofsted_date?: string | null;
ofsted_framework?: string | null;
/** The inspectorate GIAS names ("ISI", "Ofsted", "Not applicable"). Audit H4. */
inspectorate?: string | null;
/** Whether Ofsted's own data lists the school, inspected or not. Audit H4. */
ofsted_listed?: boolean | null;
}
// ============================================================================
@@ -97,8 +93,7 @@ export interface School {
// ============================================================================
export interface OfstedInspection {
/** Ofsted's event grouping (e.g. "Schools - S5"), or "Further education and skills" for FE records (audit H4). */
framework: string | null;
framework: 'OEIF' | 'ReportCard' | null;
inspection_date: string | null;
/** Start date of the report-card inspection itself (Nov 2025+); null otherwise. */
rc_inspection_date?: string | null;
@@ -148,7 +143,7 @@ export interface OfstedCurrentGrade {
export interface OfstedLatestVisit {
date: string;
kind: 'report_card' | 'graded' | 'ungraded' | 'short';
kind: 'report_card' | 'graded' | 'ungraded';
outcome: string | null;
}
+3 -9
View File
@@ -678,9 +678,7 @@ export interface OfstedListBadge {
* or an ungraded visit whose outcome names no grade): "Inspected · YYYY",
* dated by the latest visit, so a school never reads as both inspected and
* "Not yet inspected"
* - No inspection on record (audit H4): "Inspected by ISI" when GIAS names
* ISI; "Not yet inspected" only when Ofsted's own data lists the school;
* otherwise no badge, never a claim the data cannot support.
* - No inspection on record: "Not yet inspected" in grey
*/
export function buildOfstedListBadge(school: {
ofsted_grade?: 1 | 2 | 3 | 4 | null;
@@ -688,9 +686,7 @@ export function buildOfstedListBadge(school: {
ofsted_date?: string | null;
ofsted_framework?: string | null;
ofsted_rc_date?: string | null;
inspectorate?: string | null;
ofsted_listed?: boolean | null;
}): OfstedListBadge | null {
}): OfstedListBadge {
// A report card wins over any carried-forward legacy grade — signalled by
// ofsted_rc_date. ofsted_framework is the raw event grouping ("Schools -
// S5"), never "ReportCard", so it can't detect report cards.
@@ -723,9 +719,7 @@ export function buildOfstedListBadge(school: {
return { label: `Inspected${visitYear ? ` · ${visitYear}` : ''}`, cssClass: 'ofstedInspected' };
}
if (school.inspectorate === 'ISI') return { label: 'Inspected by ISI', cssClass: 'ofstedPending' };
if (school.ofsted_listed) return { label: 'Not yet inspected', cssClass: 'ofstedPending' };
return null;
return { label: 'Not yet inspected', cssClass: 'ofstedPending' };
}
// ============================================================================
+1 -1
View File
@@ -146,7 +146,7 @@ with DAG(
dbt_build_ofsted = BashOperator(
task_id="dbt_build",
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ofsted_inspections+ stg_ofsted_fe_providers+ stg_ofsted_fe_inspections+ stg_gias_ukprn+ int_ofsted_latest+ fact_ofsted_inspection+ dim_school+ --exclude stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
)
sync_typesense_ofsted = BashOperator(
@@ -72,6 +72,12 @@ class GIASEstablishmentsStream(Stream):
th.Property("Northing", th.StringType),
th.Property("StatutoryLowAge", th.StringType),
th.Property("StatutoryHighAge", th.StringType),
# Matches Ofsted's further-education providers, which some FE
# records name by UKPRN only (audit H4).
th.Property("UKPRN", th.StringType),
# "ISI" for independent schools the Independent Schools Inspectorate
# inspects; "Ofsted" or "Not applicable" otherwise (audit H4).
th.Property("InspectorateName (name)", th.StringType),
).to_dict()
def get_records(self, context):
@@ -0,0 +1,188 @@
"""Ofsted's further education and skills management information (audit H4).
Ofsted publishes FE inspection outcomes on one gov.uk page per academic year
("Further education and skills inspections and outcomes: management
information from November 2025 to August 2026"). Each page carries "as at"
files (one row per provider, with its latest inspection under the previous
framework) and cumulative files of renewed-framework inspections, i.e. report
cards ("…_10_November_2025_to_31_January_2026.csv", "…_to_28_February_2026.csv",
…). Monitoring-visit files are a different series.
Free of the Singer SDK so CI's pytest, which installs only the backend's
requirements, can load it.
"""
from __future__ import annotations
import io
import re
from datetime import date
from typing import Callable, Iterator
import pandas as pd
SEARCH_URL = "https://www.gov.uk/api/search.json"
SEARCH_QUERY = "further education and skills inspections and outcomes management information"
PAGE_TITLE_PREFIX = "Further education and skills inspections and outcomes: management information from "
# Renewed-framework (report card) inspections began on 10 November 2025.
RENEWED_FRAMEWORK_START = date(2025, 11, 1)
_MONTHS = {name: number for number, name in enumerate(
["january", "february", "march", "april", "may", "june", "july",
"august", "september", "october", "november", "december"], start=1)}
_TITLE_START = re.compile(r"from ([A-Za-z]+) (\d{4})")
_CSV_LINK = re.compile(r'https://assets\.publishing\.service\.gov\.uk/[^"\s<>]+\.csv')
_AS_AT = re.compile(r"further_education_and_skills_-_+as_at_(\d{1,2})_([A-Za-z]+)_(\d{4})\.csv$")
_PERIOD = re.compile(
r"further_education_and_skills_-_+(\d{1,2})_([A-Za-z]+)_(\d{4})_to_(\d{1,2})_([A-Za-z]+)_(\d{4})\.csv$")
# Ofsted column → Singer field.
PROVIDER_COLUMNS = {
"Provider URN": "provider_urn",
"Provider UKPRN": "provider_ukprn",
"Provider name": "provider_name",
"Provider type": "provider_type",
"Has the provider received a full inspection under the renewed EIF?": "renewed_eif_full_inspection",
"First day of inspection of latest inspection before renewed EIF": "graded_inspection_date",
"Latest overall effectiveness before renewed EIF": "overall_effectiveness",
"Date of latest short inspection before renewed EIF (if relevant)": "short_inspection_date",
}
INSPECTION_COLUMNS = {
"Provider URN": "provider_urn",
"Provider UKPRN": "provider_ukprn",
"Inspection number": "inspection_number",
"First day of inspection": "first_day_of_inspection",
"Date published": "date_published",
"Safeguarding standards": "safeguarding_standards",
}
def _date(day: str, month: str, year: str) -> date | None:
number = _MONTHS.get(month.lower())
return date(int(year), number, int(day)) if number else None
def _page_start(title: str) -> date | None:
match = _TITLE_START.search(title)
number = _MONTHS.get(match.group(1).lower()) if match else None
return date(int(match.group(2)), number, 1) if number else None
def _dated_pages(results: list[dict]) -> list[tuple[date, str]]:
pages = []
for result in results:
title = result.get("title", "")
start = _page_start(title) if title.startswith(PAGE_TITLE_PREFIX) else None
if start:
pages.append((start, result["link"]))
return sorted(pages, reverse=True)
def fe_pages(results: list[dict]) -> list[str]:
"""gov.uk paths of the FE MI pages, newest academic year first."""
return [link for _, link in _dated_pages(results)]
def renewed_framework_pages(results: list[dict]) -> list[str]:
"""The pages that can hold report cards: from November 2025 onwards."""
return [link for start, link in _dated_pages(results) if start >= RENEWED_FRAMEWORK_START]
def _latest(html: str, pattern: re.Pattern, date_groups: slice) -> str | None:
best = None
for link in set(_CSV_LINK.findall(html)):
match = pattern.search(link)
when = _date(*match.groups()[date_groups]) if match else None
if when and (best is None or when > best[0]):
best = (when, link)
return best[1] if best else None
def latest_as_at_csv(html: str) -> str | None:
"""The newest "as at" CSV on a page."""
return _latest(html, _AS_AT, slice(0, 3))
def latest_period_csv(html: str) -> str | None:
"""The renewed-framework inspections CSV with the latest end date on a page."""
return _latest(html, _PERIOD, slice(3, 6))
def providers_csv_url(pages: list[str], fetch: Callable[[str], str]) -> str | None:
"""The newest "as at" CSV, from the newest page that has one."""
for page in pages:
url = latest_as_at_csv(fetch(page))
if url:
return url
return None
def read_mi_csv(content: bytes) -> pd.DataFrame:
"""Ofsted's MI CSVs: UTF-8 with a BOM, or Windows-1252. The header is the
line naming "Provider URN"."""
try:
text = content.decode("utf-8-sig")
except UnicodeDecodeError:
text = content.decode("cp1252")
lines = text.splitlines()
start = next((i for i, line in enumerate(lines[:20]) if "Provider URN" in line), 0)
return pd.read_csv(io.StringIO("\n".join(lines[start:])), dtype=str, keep_default_na=False)
def records(df: pd.DataFrame, columns: dict[str, str], key: str) -> Iterator[dict]:
"""Singer records: the mapped columns, stripped. Rows without `key` are skipped."""
present = {source: field for source, field in columns.items() if source in df.columns}
for _, row in df.iterrows():
record = {field: str(row[source]).strip() for source, field in present.items()}
if record.get(key):
yield record
# requests' exceptions are OSErrors; a body that is not JSON is a ValueError.
_UNAVAILABLE = (OSError, ValueError)
def provider_records(
search: Callable[[], list[dict]],
fetch_page: Callable[[str], str],
fetch_csv: Callable[[str], bytes],
log: Callable[[str], None],
) -> Iterator[dict]:
"""The newest "as at" file's provider records. gov.uk being unreachable
yields nothing, logged: the FE streams share the monthly Ofsted extract
with the school MI and must not fail it."""
try:
url = providers_csv_url(fe_pages(search()), fetch_page)
if not url:
log("No Ofsted FE 'as at' file found")
return
df = read_mi_csv(fetch_csv(url))
except _UNAVAILABLE as exc:
log(f"Ofsted FE providers MI unavailable: {exc}")
return
yield from records(df, PROVIDER_COLUMNS, "provider_urn")
def inspection_records(
search: Callable[[], list[dict]],
fetch_page: Callable[[str], str],
fetch_csv: Callable[[str], bytes],
log: Callable[[str], None],
) -> Iterator[dict]:
"""Renewed-framework inspection records from every page since November
2025. A page that cannot be read is logged and skipped; the others load."""
try:
pages = renewed_framework_pages(search())
except _UNAVAILABLE as exc:
log(f"Ofsted FE inspections MI unavailable: {exc}")
return
for page in pages:
try:
url = latest_period_csv(fetch_page(page))
if not url:
log(f"No renewed-framework FE inspections file on {page}")
continue
df = read_mi_csv(fetch_csv(url))
except _UNAVAILABLE as exc:
log(f"Ofsted FE inspections MI unavailable for {page}: {exc}")
continue
yield from records(df, INSPECTION_COLUMNS, "inspection_number")
@@ -10,6 +10,15 @@ import requests
from singer_sdk import Stream, Tap
from singer_sdk import typing as th
from tap_uk_ofsted.fe_mi import (
INSPECTION_COLUMNS,
PROVIDER_COLUMNS,
SEARCH_QUERY,
SEARCH_URL,
inspection_records,
provider_records,
)
GOV_UK_PAGE = (
"https://www.gov.uk/government/statistical-data-sets/"
"monthly-management-information-ofsteds-school-inspections-outcomes"
@@ -304,6 +313,62 @@ class OfstedInspectionsStream(Stream):
self.logger.error("Could not discover Ofsted independent MI download URL")
def _fe_search_results() -> list[dict]:
"""gov.uk search results for Ofsted's FE MI pages (one per academic year)."""
resp = requests.get(SEARCH_URL, params={"q": SEARCH_QUERY, "count": 50}, timeout=60)
resp.raise_for_status()
return resp.json().get("results", [])
def _gov_uk_page(path: str) -> str:
url = path if path.startswith("http") else "https://www.gov.uk" + path
resp = requests.get(url, timeout=60)
resp.raise_for_status()
return resp.text
def _gov_uk_file(url: str) -> bytes:
resp = requests.get(url, timeout=120)
resp.raise_for_status()
return resp.content
class OfstedFEProvidersStream(Stream):
"""Ofsted's further education and skills providers: the newest "as at"
file, one row per provider with its latest inspection under the previous
framework (audit H4). Page and file selection: fe_mi.py."""
name = "ofsted_fe_providers"
primary_keys = ["provider_urn"]
replication_key = None
schema = th.PropertiesList(
th.Property("provider_urn", th.StringType, required=True),
*[th.Property(field, th.StringType) for field in PROVIDER_COLUMNS.values() if field != "provider_urn"],
).to_dict()
def get_records(self, context):
yield from provider_records(_fe_search_results, _gov_uk_page, _gov_uk_file, self.logger.error)
class OfstedFEInspectionsStream(Stream):
"""Ofsted's renewed-framework FE inspections (report cards), from the
latest cumulative file on every page since November 2025, so a report
card survives the next academic year's page (audit H4)."""
name = "ofsted_fe_inspections"
primary_keys = ["inspection_number"]
replication_key = None
schema = th.PropertiesList(
th.Property("inspection_number", th.StringType, required=True),
*[th.Property(field, th.StringType) for field in INSPECTION_COLUMNS.values() if field != "inspection_number"],
).to_dict()
def get_records(self, context):
yield from inspection_records(_fe_search_results, _gov_uk_page, _gov_uk_file, self.logger.warning)
class TapUKOfsted(Tap):
"""Singer tap for UK Ofsted Management Information."""
@@ -315,7 +380,7 @@ class TapUKOfsted(Tap):
).to_dict()
def discover_streams(self):
return [OfstedInspectionsStream(self)]
return [OfstedInspectionsStream(self), OfstedFEProvidersStream(self), OfstedFEInspectionsStream(self)]
if __name__ == "__main__":
+41 -3
View File
@@ -24,9 +24,9 @@ DAG_ID = re.compile(r'dag_id="([a-z0-9_]+)"')
DAILY = 'school_data_daily'
# dim_school reads int_ofsted_latest only when the relation exists
# (adapter.get_relation), so a missing table is not a failure.
OPTIONAL_PARENTS = {'int_ofsted_latest'}
# dim_school reads int_ofsted_latest and int_ofsted_listed only when the
# relations exist (adapter.get_relation), so a missing table is not a failure.
OPTIONAL_PARENTS = {'int_ofsted_latest', 'int_ofsted_listed'}
def model_parents():
@@ -96,3 +96,41 @@ def test_selected_models_only_read_models_that_exist(dag_id):
missing = {model: sorted(PARENTS[model] - available) for model in sorted(selected)
if PARENTS[model] - available}
assert missing == {}, f'{dag_id} builds models whose parents it never builds: {missing}'
TABLE = re.compile(r"materialized\s*=\s*['\"]table['\"]")
def views():
"""Staging and intermediate models are views unless configured as tables."""
out = set()
for path in MODELS.rglob('*.sql'):
layer = path.relative_to(MODELS).parts[0]
if layer in ('staging', 'intermediate') and not TABLE.search(path.read_text()):
out.add(path.stem)
return out
VIEWS = views()
@pytest.mark.parametrize('dag_id', sorted(BUILDS))
def test_rebuilt_relations_do_not_drop_views_the_build_reads(dag_id):
"""dbt rebuilds a relation by dropping the old one with CASCADE, which drops
every view built on it. A view dropped that way and not rebuilt by the same
build is gone when that build reads it: the daily GIAS build rebuilt
stg_gias_ukprn and dropped the Ofsted views dim_school reads (audit H4)."""
selected = selected_models(BUILDS[dag_id], PARENTS)
children = defaultdict(set)
for model, ps in PARENTS.items():
for p in ps:
children[p].add(model)
dropped, stack = set(), list(selected)
while stack:
for child in children[stack.pop()]:
if child in VIEWS and child not in dropped:
dropped.add(child)
stack.append(child)
dropped -= selected
broken = {m: sorted(PARENTS[m] & dropped) for m in sorted(selected) if PARENTS[m] & dropped}
assert broken == {}, f'{dag_id} drops views it then reads: {broken}'
@@ -0,0 +1,61 @@
"""Every field the GIAS establishments stream declares must be a column of the
extract GIAS publishes, or it silently loads as null. H4 adds UKPRN (matches
Ofsted's further-education providers) and InspectorateName (name) ("ISI" for
about 1,340 independent schools). HEADER is GIAS's own, from
edubasealldata20261013.csv.
"""
import re
from pathlib import Path
TAP = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-gias'
/ 'tap_uk_gias' / 'tap.py')
HEADER = (
'URN', 'LA (code)', 'LA (name)', 'EstablishmentNumber', 'EstablishmentName',
'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)', 'EstablishmentTypeGroup (code)',
'EstablishmentTypeGroup (name)', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)',
'ReasonEstablishmentOpened (code)', 'ReasonEstablishmentOpened (name)', 'OpenDate',
'ReasonEstablishmentClosed (code)', 'ReasonEstablishmentClosed (name)', 'CloseDate',
'PhaseOfEducation (code)', 'PhaseOfEducation (name)', 'StatutoryLowAge', 'StatutoryHighAge',
'Boarders (code)', 'Boarders (name)', 'NurseryProvision (name)', 'OfficialSixthForm (code)',
'OfficialSixthForm (name)', 'Gender (code)', 'Gender (name)', 'ReligiousCharacter (code)',
'ReligiousCharacter (name)', 'ReligiousEthos (name)', 'Diocese (code)', 'Diocese (name)',
'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)', 'SchoolCapacity', 'SpecialClasses (code)',
'SpecialClasses (name)', 'CensusDate', 'NumberOfPupils', 'NumberOfBoys', 'NumberOfGirls',
'PercentageFSM', 'TrustSchoolFlag (code)', 'TrustSchoolFlag (name)', 'Trusts (code)',
'Trusts (name)', 'SchoolSponsorFlag (name)', 'SchoolSponsors (name)', 'FederationFlag (name)',
'Federations (code)', 'Federations (name)', 'UKPRN', 'FEHEIdentifier',
'FurtherEducationType (name)', 'LastChangedDate', 'Street', 'Locality', 'Address3', 'Town',
'County (name)', 'Postcode', 'SchoolWebsite', 'TelephoneNum', 'HeadTitle (name)',
'HeadFirstName', 'HeadLastName', 'HeadPreferredJobTitle', 'BSOInspectorateName (name)',
'InspectorateReport', 'DateOfLastInspectionVisit', 'NextInspectionVisit', 'TeenMoth (name)',
'TeenMothPlaces', 'CCF (name)', 'SENPRU (name)', 'EBD (name)', 'PlacesPRU', 'FTProv (name)',
'EdByOther (name)', 'Section41Approved (name)', 'SEN1 (name)', 'SEN2 (name)', 'SEN3 (name)',
'SEN4 (name)', 'SEN5 (name)', 'SEN6 (name)', 'SEN7 (name)', 'SEN8 (name)', 'SEN9 (name)',
'SEN10 (name)', 'SEN11 (name)', 'SEN12 (name)', 'SEN13 (name)',
'TypeOfResourcedProvision (name)', 'ResourcedProvisionOnRoll', 'ResourcedProvisionCapacity',
'SenUnitOnRoll', 'SenUnitCapacity', 'GOR (code)', 'GOR (name)',
'DistrictAdministrative (code)', 'DistrictAdministrative (name)', 'AdministrativeWard (code)',
'AdministrativeWard (name)', 'ParliamentaryConstituency (code)',
'ParliamentaryConstituency (name)', 'UrbanRural (code)', 'UrbanRural (name)',
'GSSLACode (name)', 'Easting', 'Northing', 'MSOA (name)', 'LSOA (name)',
'InspectorateName (name)', 'SENStat', 'SENNoStat', 'BoardingEstablishment (name)', 'PropsName',
'PreviousLA (code)', 'PreviousLA (name)', 'PreviousEstablishmentNumber', 'Country (name)',
'UPRN', 'SiteName', 'QABName (code)', 'QABName (name)', 'EstablishmentAccredited (code)',
'EstablishmentAccredited (name)', 'QABReport', 'CHNumber', 'MSOA (code)', 'LSOA (code)', 'FSM',
'AccreditationExpiryDate',
)
def declared():
source = TAP.read_text()
block = source[source.index('class GIASEstablishmentsStream'):source.index('class GIASLinksStream')]
return re.findall(r'th\.Property\("([^"]+)"', block)
def test_every_declared_field_is_a_gias_column():
assert set(declared()) - set(HEADER) == set()
def test_the_inspection_fields_are_declared():
assert {'UKPRN', 'InspectorateName (name)'} <= set(declared())
+165
View File
@@ -0,0 +1,165 @@
"""Ofsted publishes further-education inspection outcomes on a gov.uk page per
academic year, separate from the school MI the tap already reads (audit H4).
The links below are the real ones from the 2025/26 page; the headers are
Ofsted's own.
"""
import importlib.util
from pathlib import Path
import pandas as pd
import pytest
MODULE = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-ofsted'
/ 'tap_uk_ofsted' / 'fe_mi.py')
A = 'https://assets.publishing.service.gov.uk/media/'
PAGE_2025_26 = f'''
<a href="{A}6989b16dc106cc5501b6e4c3/Management_information_-_further_education_and_skills_-_as_at_31_January_2026.csv">x</a>
<a href="{A}6989b18cd3f57710b50a9b92/Management_information_-_further_education_and_skills_-_10_November_2025_to_31_January_2026.csv">x</a>
<a href="{A}69aec8d6d620c14fa183eeb6/Management_information_-_further_education_and_skills_-__10_November_2025_to_28_February_2026.csv">x</a>
<a href="{A}69aec8fcc78869bf8eb8a54a/Management_information_-_further_education_and_skills_-_as_at_28_February_2026.csv">x</a>
<a href="{A}69aef091bde9c3f213c89a1e/Management_information_-_further_education_and_skills_-_as_at_31_January_2026.ods">x</a>
<a href="{A}6a9fec1e92e72b8ac437eef3/Management_information_-_further_education_and_skills_-_as_at_31_August_2026.csv">x</a>
<a href="{A}6a9fed6b5f6e942efe37eed4/Management_information_-_new_provider_monitoring_inspections_-_10_November_2025_to_31_August_2026.csv">x</a>
<a href="{A}6a9fed815f6e942efe37eed5/Management_information_-_monitoring_inspections_-_10_November_2025_to_31_August_2026.csv">x</a>
<a href="{A}6a9feda65f6e942efe37eed6/Management_information_-_further_education_and_skills_-_10_November_2025_to_31_August_2026.csv">x</a>
'''
SEARCH_RESULTS = [
{'title': 'Further education and skills inspections and outcomes: management information from September 2024 to August 2025',
'link': '/government/statistical-data-sets/fe-2024-25'},
{'title': 'Further education and skills inspections and outcomes: management information from November 2025 to August 2026',
'link': '/government/statistical-data-sets/fe-2025-26'},
{'title': 'Further education and skills inspections and outcomes: management information from September 2023 to August 2024',
'link': '/government/statistical-data-sets/fe-2023-24'},
{'title': 'Monthly management information: Ofsted\'s school inspections outcomes',
'link': '/government/statistical-data-sets/schools'},
]
PROVIDER_HEADER = (
'Provider URN', 'Provider UKPRN', 'Provider name', 'Provider type', 'Provider group',
'Local authority', 'Region', 'Ofsted region',
'Has the provider received a full inspection under the renewed EIF?',
'Has the provider received a new provider monitoring inspection?',
'Date of latest short inspection before renewed EIF (if relevant)',
'Inspection number of latest full inspection before renewed EIF',
'First day of inspection of latest inspection before renewed EIF',
'Latest overall effectiveness before renewed EIF',
'Latest quality of education before renewed EIF',
'Latest behaviour and attitudes before renewed EIF',
'Latest personal development before renewed EIF',
'Latest leadership and management before renewed EIF',
'Latest contribution to meeting skills needs before renewed EIF',
'Latest safeguarding before renewed EIF', '',
)
INSPECTION_HEADER = (
'Provider URN', 'Provider UKPRN', 'Provider name', 'Provider type', 'Provider group',
'Local authority', 'Region', 'Ofsted region', 'Inspection number', 'First day of inspection',
'Date published', 'Safeguarding standards', 'Inclusion', 'Leadership and governance',
'Contribution to meeting skills needs',
'Education programmes for young people - curriculum, teaching and training',
'Education programmes for young people - achievement',
'Education programmes for young people - participation and development',
'Adult learning programmes - curriculum, teaching and training',
'Adult learning programmes - achievement',
'Adult learning programmes - participation and development',
'Apprenticeships - curriculum, teaching and training', 'Apprenticeships - achievement',
'Apprenticeships - participation and development',
'Provision for learners and apprentices with high needs - curriculum, teaching and training',
'Provision for learners and apprentices with high needs - achievement',
'Provision for learners and apprentices with high needs - participation and development', '',
)
@pytest.fixture
def fe():
spec = importlib.util.spec_from_file_location('fe_mi', MODULE)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_the_newest_as_at_file_is_chosen(fe):
assert fe.latest_as_at_csv(PAGE_2025_26).endswith('_as_at_31_August_2026.csv')
def test_the_latest_cumulative_inspections_file_is_chosen_not_a_monitoring_one(fe):
assert fe.latest_period_csv(PAGE_2025_26) == (
A + '6a9feda65f6e942efe37eed6/Management_information_-_further_education_and_skills_'
'-_10_November_2025_to_31_August_2026.csv')
def test_a_doubled_underscore_file_name_still_parses(fe):
page = '\n'.join(line for line in PAGE_2025_26.splitlines() if 'August' not in line)
assert fe.latest_period_csv(page).endswith('__10_November_2025_to_28_February_2026.csv')
def test_monitoring_files_alone_give_nothing(fe):
page = '\n'.join(line for line in PAGE_2025_26.splitlines() if 'monitoring' in line)
assert fe.latest_period_csv(page) is None
assert fe.latest_as_at_csv(page) is None
def test_pages_are_the_fe_series_newest_first(fe):
assert fe.fe_pages(SEARCH_RESULTS) == [
'/government/statistical-data-sets/fe-2025-26',
'/government/statistical-data-sets/fe-2024-25',
'/government/statistical-data-sets/fe-2023-24',
]
def test_report_cards_are_read_only_from_renewed_framework_pages(fe):
# Earlier pages hold previous-framework inspection lists, not report cards.
assert fe.renewed_framework_pages(SEARCH_RESULTS) == ['/government/statistical-data-sets/fe-2025-26']
def test_providers_fall_back_to_the_previous_page(fe):
pages = {'/new': '<p>No files yet</p>', '/old': PAGE_2025_26}
assert fe.providers_csv_url(['/new', '/old'], pages.__getitem__).endswith('_as_at_31_August_2026.csv')
@pytest.mark.parametrize('columns, header', [('PROVIDER_COLUMNS', PROVIDER_HEADER),
('INSPECTION_COLUMNS', INSPECTION_HEADER)])
def test_every_mapped_column_is_in_ofsteds_file(fe, columns, header):
assert set(getattr(fe, columns)) - set(header) == set()
def test_csv_reading_and_records(fe):
content = ('' + ','.join(f'"{c}"' for c in PROVIDER_HEADER[:14]) + '\n'
+ '145175,10005429,Queen Mary\'s College,Sixth form college,Sixth form colleges,Hampshire,'
'South East,South East,No,No,-,10227183,06/12/2022,2\n'
+ ',,Blank provider,,,,,,,,,,,\n').encode('utf-8')
rows = list(fe.records(fe.read_mi_csv(content), fe.PROVIDER_COLUMNS, 'provider_urn'))
assert rows == [{
'provider_urn': '145175', 'provider_ukprn': '10005429', 'provider_name': "Queen Mary's College",
'provider_type': 'Sixth form college', 'renewed_eif_full_inspection': 'No',
'graded_inspection_date': '06/12/2022', 'overall_effectiveness': '2', 'short_inspection_date': '-',
}]
def test_a_gov_uk_outage_yields_no_providers_rather_than_failing(fe):
# The FE streams share the monthly Ofsted extract with the school MI; an
# error here must not fail that extract (review of H4).
def down():
raise ConnectionError('gov.uk unreachable')
logged = []
assert list(fe.provider_records(down, lambda page: '', lambda url: b'', logged.append)) == []
assert logged
def test_one_unreachable_page_does_not_stop_the_other_report_cards(fe):
results = [{'title': fe.PAGE_TITLE_PREFIX + 'September 2026 to August 2027', 'link': '/fe-2026-27'},
{'title': fe.PAGE_TITLE_PREFIX + 'November 2025 to August 2026', 'link': '/fe-2025-26'}]
csv = (','.join(INSPECTION_HEADER[:12]) + '\n'
+ '143526,10028480,The Autism Project,,,,,,10427540,12/05/2026,22/07/2026,Met\n').encode()
def page(path):
if path == '/fe-2026-27':
raise ConnectionError('timeout')
return PAGE_2025_26
logged = []
rows = list(fe.inspection_records(lambda: results, page, lambda url: csv, logged.append))
assert [r['inspection_number'] for r in rows] == ['10427540']
assert logged
@@ -0,0 +1,98 @@
-- Intermediate model: Ofsted further education and skills status per URN
-- (audit H4), in int_ofsted_latest's columns so it can be unioned there.
-- A provider matches a school by Provider URN, else by UKPRN (stg_gias_ukprn).
-- Rule, in the C1 status rule's terms
-- (docs/superpowers/specs/2026-10-06-inspection-status-beyond-ofsted-schools-design.md):
-- a report card on or after the previous-framework inspection wins and
-- leaves no grade in force; otherwise a previous-framework grade 1-4 is the
-- current grade, dated by that inspection; the latest visit is the later of
-- it and any short inspection (whose outcome the MI does not give).
with gias as (
select urn, ukprn from {{ ref('stg_gias_ukprn') }}
),
providers as (
select distinct on (g.urn)
g.urn,
p.graded_inspection_date,
p.overall_effectiveness,
p.short_inspection_date
from gias g
join {{ ref('stg_ofsted_fe_providers') }} p
on p.provider_urn = g.urn
or (g.ukprn is not null and p.provider_ukprn = g.ukprn)
order by g.urn, (p.provider_urn = g.urn) desc, p.graded_inspection_date desc nulls last
),
report_cards as (
select distinct on (g.urn)
g.urn,
c.rc_inspection_date,
c.rc_safeguarding_met
from gias g
join {{ ref('stg_ofsted_fe_inspections') }} c
on c.provider_urn = g.urn
or (g.ukprn is not null and c.provider_ukprn = g.ukprn)
order by g.urn, (c.provider_urn = g.urn) desc, c.rc_inspection_date desc, c.inspection_number desc
),
combined as (
select
coalesce(p.urn, c.urn) as urn,
p.graded_inspection_date,
p.overall_effectiveness,
p.short_inspection_date,
c.rc_inspection_date as card_date,
c.rc_safeguarding_met as card_safeguarding_met
from providers p
full outer join report_cards c on c.urn = p.urn
),
decided as (
select
*,
card_date is not null
and card_date >= coalesce(greatest(graded_inspection_date, short_inspection_date), card_date)
as report_card_latest
from combined
)
select
urn,
case when report_card_latest then card_date
else greatest(graded_inspection_date, short_inspection_date) end as latest_visit_date,
case when report_card_latest then 'report_card'
when short_inspection_date > graded_inspection_date then 'short'
else 'graded' end as latest_visit_kind,
null::text as latest_visit_outcome,
case when not report_card_latest then overall_effectiveness end as current_grade,
case when not report_card_latest then graded_inspection_date end as current_grade_date,
case when not report_card_latest then 'graded' end as current_grade_basis,
graded_inspection_date,
null::date as ungraded_inspection_date,
coalesce(graded_inspection_date, card_date) as inspection_date,
null::text as inspection_type,
'Further education and skills'::text as framework,
overall_effectiveness,
null::integer as quality_of_education,
null::integer as behaviour_attitudes,
null::integer as personal_development,
null::integer as leadership_management,
null::integer as early_years_provision,
null::integer as sixth_form_provision,
null::text as ungraded_outcome,
null::integer as ungraded_grade,
case when report_card_latest then card_safeguarding_met end as rc_safeguarding_met,
null::integer as rc_inclusion,
null::integer as rc_curriculum_teaching,
null::integer as rc_achievement,
null::integer as rc_attendance_behaviour,
null::integer as rc_personal_development,
null::integer as rc_leadership_governance,
null::integer as rc_early_years,
null::integer as rc_sixth_form,
case when report_card_latest then card_date end as rc_inspection_date,
null::text as report_url
from decided
where report_card_latest or overall_effectiveness between 1 and 4
@@ -0,0 +1,128 @@
version: 2
unit_tests:
- name: int_ofsted_fe_grade_only
description: Queen Mary's College (145175), Good on 6 December 2022, no report card.
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 145175, ukprn: '10005429'}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 145175, provider_ukprn: '10005429', graded_inspection_date: '2022-12-06', overall_effectiveness: 2}
- input: ref('stg_ofsted_fe_inspections')
rows: []
expect:
rows:
- {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, rc_inspection_date: null, framework: 'Further education and skills'}
- name: int_ofsted_fe_short_inspection_is_a_visit_not_a_grade
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 1, ukprn: null}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 1, graded_inspection_date: '2019-03-05', overall_effectiveness: 2, short_inspection_date: '2023-02-01'}
- input: ref('stg_ofsted_fe_inspections')
rows: []
expect:
rows:
- {urn: 1, latest_visit_date: '2023-02-01', latest_visit_kind: short, current_grade: 2, current_grade_date: '2019-03-05', current_grade_basis: graded}
- name: int_ofsted_fe_newer_report_card_wins
description: The Autism Project - CareTrade (143526). Good 2021; report card 12 May 2026, safeguarding met.
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 143526, ukprn: '10000002'}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 143526, provider_ukprn: '10000002', graded_inspection_date: '2021-11-16', overall_effectiveness: 2}
- input: ref('stg_ofsted_fe_inspections')
rows:
- {inspection_number: '10380001', provider_urn: 143526, provider_ukprn: '10000002', rc_inspection_date: '2026-05-12', rc_safeguarding_met: true}
expect:
rows:
- {urn: 143526, latest_visit_date: '2026-05-12', latest_visit_kind: report_card, current_grade: null, current_grade_date: null, current_grade_basis: null, rc_inspection_date: '2026-05-12', rc_safeguarding_met: true, overall_effectiveness: 2}
- name: int_ofsted_fe_report_card_only
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 150042, ukprn: null}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 150042, graded_inspection_date: null, overall_effectiveness: null}
- input: ref('stg_ofsted_fe_inspections')
rows:
- {inspection_number: '10380002', provider_urn: 150042, rc_inspection_date: '2026-01-20', rc_safeguarding_met: false}
expect:
rows:
- {urn: 150042, latest_visit_date: '2026-01-20', latest_visit_kind: report_card, current_grade: null, rc_inspection_date: '2026-01-20', rc_safeguarding_met: false}
- name: int_ofsted_fe_listed_never_inspected_has_no_row
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 130645, ukprn: null}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 130645, graded_inspection_date: null, overall_effectiveness: null}
- input: ref('stg_ofsted_fe_inspections')
rows: []
expect:
rows: []
- name: int_ofsted_fe_matches_by_ukprn_when_the_urn_differs
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 5, ukprn: '10099999'}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 9999999, provider_ukprn: '10099999', graded_inspection_date: '2024-01-30', overall_effectiveness: 3}
- input: ref('stg_ofsted_fe_inspections')
rows: []
expect:
rows:
- {urn: 5, current_grade: 3, current_grade_date: '2024-01-30'}
- name: int_ofsted_fe_prefers_the_urn_match
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 6, ukprn: '10088888'}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 6, provider_ukprn: '10077777', graded_inspection_date: '2023-05-01', overall_effectiveness: 1}
- {provider_urn: 8888888, provider_ukprn: '10088888', graded_inspection_date: '2024-05-01', overall_effectiveness: 4}
- input: ref('stg_ofsted_fe_inspections')
rows: []
expect:
rows:
- {urn: 6, current_grade: 1, current_grade_date: '2023-05-01'}
- name: int_ofsted_fe_report_card_older_than_grade_is_ignored
description: Bad data guard. A report card dated before the graded inspection cannot be the latest, and a row must never carry both a grade and a report card.
model: int_ofsted_fe_latest
given:
- input: ref('stg_gias_ukprn')
rows:
- {urn: 7, ukprn: null}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 7, graded_inspection_date: '2026-03-01', overall_effectiveness: 2}
- input: ref('stg_ofsted_fe_inspections')
rows:
- {inspection_number: '1', provider_urn: 7, rc_inspection_date: '2026-01-10', rc_safeguarding_met: true}
expect:
rows:
- {urn: 7, latest_visit_kind: graded, current_grade: 2, rc_inspection_date: null, rc_safeguarding_met: null}
@@ -3,6 +3,8 @@
-- inspection) and the overall grade still in force, if any. A grade is dated
-- by the inspection that awarded or confirmed it, never by a later visit.
-- Rule and examples: docs/superpowers/specs/2026-10-05-ofsted-current-status-design.md
-- Schools with no school-framework record take their further education and
-- skills status from int_ofsted_fe_latest (same columns, same order; audit H4).
with inspections as (
select
@@ -62,51 +64,64 @@ graded as (
then 'graded_earlier'
end as grade_case
from latest
),
school_status as (
select
urn,
latest_visit_date,
latest_visit_kind,
case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome,
case grade_case
when 'confirmed' then ungraded_grade
when 'graded_latest' then overall_effectiveness
when 'graded_earlier' then overall_effectiveness
end as current_grade,
case grade_case
when 'confirmed' then ungraded_inspection_date
when 'graded_latest' then graded_inspection_date
when 'graded_earlier' then graded_inspection_date
end as current_grade_date,
case grade_case
when 'confirmed' then 'confirmed'
when 'graded_latest' then 'graded'
when 'graded_earlier' then 'graded'
end as current_grade_basis,
graded_inspection_date,
ungraded_inspection_date,
inspection_date,
inspection_type,
framework,
overall_effectiveness,
quality_of_education,
behaviour_attitudes,
personal_development,
leadership_management,
early_years_provision,
sixth_form_provision,
ungraded_outcome,
ungraded_grade,
rc_safeguarding_met,
rc_inclusion,
rc_curriculum_teaching,
rc_achievement,
rc_attendance_behaviour,
rc_personal_development,
rc_leadership_governance,
rc_early_years,
rc_sixth_form,
rc_inspection_date,
report_url
from graded
),
fe_status as (
-- Same columns in the same order as school_status.
select f.*
from {{ ref('int_ofsted_fe_latest') }} f
where not exists (select 1 from school_status s where s.urn = f.urn)
)
select
urn,
latest_visit_date,
latest_visit_kind,
case when latest_visit_kind = 'ungraded' then ungraded_outcome end as latest_visit_outcome,
case grade_case
when 'confirmed' then ungraded_grade
when 'graded_latest' then overall_effectiveness
when 'graded_earlier' then overall_effectiveness
end as current_grade,
case grade_case
when 'confirmed' then ungraded_inspection_date
when 'graded_latest' then graded_inspection_date
when 'graded_earlier' then graded_inspection_date
end as current_grade_date,
case grade_case
when 'confirmed' then 'confirmed'
when 'graded_latest' then 'graded'
when 'graded_earlier' then 'graded'
end as current_grade_basis,
graded_inspection_date,
ungraded_inspection_date,
inspection_date,
inspection_type,
framework,
overall_effectiveness,
quality_of_education,
behaviour_attitudes,
personal_development,
leadership_management,
early_years_provision,
sixth_form_provision,
ungraded_outcome,
ungraded_grade,
rc_safeguarding_met,
rc_inclusion,
rc_curriculum_teaching,
rc_achievement,
rc_attendance_behaviour,
rc_personal_development,
rc_leadership_governance,
rc_early_years,
rc_sixth_form,
rc_inspection_date,
report_url
from graded
select * from school_status
union all
select * from fe_status
@@ -8,6 +8,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 102408, graded_inspection_date: '2025-06-17', ungraded_inspection_date: '2020-02-06', overall_effectiveness: null, ungraded_grade: 2, ungraded_outcome: 'School remains Good'}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 102408, latest_visit_date: '2025-06-17', latest_visit_kind: graded, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null}
@@ -18,6 +20,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2019-06-01', current_grade_basis: graded}
@@ -29,6 +33,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 104762, graded_inspection_date: '2020-01-07', ungraded_inspection_date: '2024-07-18', overall_effectiveness: 2, ungraded_grade: 2, ungraded_outcome: 'School remains Good'}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 104762, latest_visit_date: '2024-07-18', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: 2, current_grade_date: '2024-07-18', current_grade_basis: confirmed}
@@ -40,6 +46,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 139888, graded_inspection_date: '2020-03-03', ungraded_inspection_date: '2025-05-21', overall_effectiveness: 2, ungraded_grade: null, ungraded_outcome: 'Standards maintained'}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 139888, latest_visit_date: '2025-05-21', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: 2, current_grade_date: '2020-03-03', current_grade_basis: graded}
@@ -51,6 +59,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 136454, ungraded_inspection_date: '2024-11-13', ungraded_grade: null, ungraded_outcome: 'Standards maintained'}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 136454, latest_visit_date: '2024-11-13', latest_visit_kind: ungraded, latest_visit_outcome: 'Standards maintained', current_grade: null, current_grade_date: null, current_grade_basis: null}
@@ -62,6 +72,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 110048, ungraded_inspection_date: '2023-10-05', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2026-05-06', rc_inclusion: 3}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 110048, latest_visit_date: '2026-05-06', latest_visit_kind: report_card, latest_visit_outcome: null, current_grade: null, current_grade_date: null, current_grade_basis: null}
@@ -74,6 +86,8 @@ unit_tests:
rows:
- {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3}
- {urn: 138186, graded_inspection_date: '2023-06-13', overall_effectiveness: 3, rc_inspection_date: '2026-06-02', rc_inclusion: 3}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 138186, latest_visit_date: '2026-06-02', latest_visit_kind: report_card, current_grade: null}
@@ -84,6 +98,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 2, graded_inspection_date: '2024-03-01', ungraded_inspection_date: '2024-03-01', overall_effectiveness: 1, ungraded_grade: 2, ungraded_outcome: 'School remains Good'}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 2, latest_visit_kind: graded, current_grade: 1, current_grade_basis: graded}
@@ -94,6 +110,8 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 3, graded_inspection_date: '2018-05-01', overall_effectiveness: 9}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 3, latest_visit_kind: graded, current_grade: null}
@@ -108,6 +126,34 @@ unit_tests:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 4, ungraded_inspection_date: '2026-03-02', ungraded_grade: 2, ungraded_outcome: 'School remains Good', rc_inspection_date: '2025-12-01', rc_inclusion: 3}
- input: ref('int_ofsted_fe_latest')
rows: []
expect:
rows:
- {urn: 4, latest_visit_date: '2026-03-02', latest_visit_kind: ungraded, latest_visit_outcome: 'School remains Good', current_grade: null, current_grade_date: null, current_grade_basis: null}
- name: fe_status_fills_a_school_with_no_school_record
description: Queen Mary's College (145175) has no school-framework record; its FE status passes through (audit H4).
model: int_ofsted_latest
given:
- input: ref('stg_ofsted_inspections')
rows: []
- input: ref('int_ofsted_fe_latest')
rows:
- {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, framework: 'Further education and skills'}
expect:
rows:
- {urn: 145175, latest_visit_date: '2022-12-06', latest_visit_kind: graded, current_grade: 2, current_grade_date: '2022-12-06', current_grade_basis: graded, framework: 'Further education and skills'}
- name: a_school_record_wins_over_an_fe_record
model: int_ofsted_latest
given:
- input: ref('stg_ofsted_inspections')
rows:
- {urn: 1, graded_inspection_date: '2019-06-01', overall_effectiveness: 2, framework: 'Schools - S5'}
- input: ref('int_ofsted_fe_latest')
rows:
- {urn: 1, latest_visit_date: '2026-05-12', latest_visit_kind: report_card, rc_inspection_date: '2026-05-12', framework: 'Further education and skills'}
expect:
rows:
- {urn: 1, latest_visit_date: '2019-06-01', latest_visit_kind: graded, current_grade: 2, framework: 'Schools - S5'}
@@ -0,0 +1,21 @@
-- Intermediate model: every URN Ofsted's own data lists, inspected or not
-- (audit H4). Only these may be called "Not yet inspected". Reads the raw
-- school MI because stg_ofsted_inspections drops rows with no inspection date.
with school_mi as (
select distinct cast(urn as integer) as urn
from {{ source('raw', 'ofsted_inspections') }}
where urn is not null
),
fe as (
select distinct g.urn
from {{ ref('stg_gias_ukprn') }} g
join {{ ref('stg_ofsted_fe_providers') }} p
on p.provider_urn = g.urn
or (g.ukprn is not null and p.provider_ukprn = g.ukprn)
)
select urn from school_mi
union
select urn from fe
@@ -0,0 +1,29 @@
version: 2
unit_tests:
- name: int_ofsted_listed_includes_uninspected_schools_and_fe_providers
description: >
Abbas and Templecombe (150031) is in Ofsted's school MI with no inspection;
Exeter and North Devon Colleges Group (130645) is in the FE MI with none;
a UKPRN-only FE match counts too. A school in neither is absent (audit H4).
model: int_ofsted_listed
given:
- input: source('raw', 'ofsted_inspections')
rows:
- {urn: '150031', inspection_date: 'NULL'}
- {urn: '102408', inspection_date: '17/06/2025'}
- input: ref('stg_gias_ukprn')
rows:
- {urn: 130645, ukprn: null}
- {urn: 5, ukprn: '10099999'}
- {urn: 133058, ukprn: null}
- input: ref('stg_ofsted_fe_providers')
rows:
- {provider_urn: 130645, provider_ukprn: null}
- {provider_urn: 9999999, provider_ukprn: '10099999'}
expect:
rows:
- {urn: 150031}
- {urn: 102408}
- {urn: 130645}
- {urn: 5}
@@ -138,7 +138,7 @@ models:
tests:
- not_null
- accepted_values:
values: ['report_card', 'graded', 'ungraded']
values: ['report_card', 'graded', 'ungraded', 'short']
- name: current_grade
tests:
- accepted_values:
+21 -3
View File
@@ -1,9 +1,10 @@
-- Mart: Canonical school dimension — one row per active URN
-- int_ofsted_latest is only ref()'d inside a conditional block below (it may not
-- exist before the Ofsted pipeline first runs), so declare the dependency
-- explicitly for dbt's DAG.
-- int_ofsted_latest and int_ofsted_listed are only ref()'d inside conditional
-- blocks below (they may not exist before the Ofsted pipeline first runs), so
-- declare the dependencies explicitly for dbt's DAG.
-- depends_on: {{ ref('int_ofsted_latest') }}
-- depends_on: {{ ref('int_ofsted_listed') }}
with schools as (
select * from {{ ref('stg_gias_establishments') }}
@@ -15,6 +16,12 @@ with schools as (
identifier='int_ofsted_latest'
) %}
{% set listed_relation = adapter.get_relation(
database=target.database,
schema='intermediate',
identifier='int_ofsted_listed'
) %}
select
s.urn,
s.local_authority_code * 1000 + s.establishment_number as laestab,
@@ -70,6 +77,14 @@ select
s.status_code,
s.nursery_provision,
s.admissions_policy_code,
-- The inspectorate GIAS names ("ISI" for most independent schools) and
-- whether Ofsted's own data lists the school, inspected or not (audit H4).
s.inspectorate_name as inspectorate,
{% if listed_relation is not none %}
l.urn is not null as ofsted_listed,
{% else %}
null::boolean as ofsted_listed,
{% endif %}
-- Latest Ofsted (populated after monthly Ofsted pipeline runs). The grade
-- still in force and the latest visit, from int_ofsted_latest — never a
@@ -88,6 +103,9 @@ from schools s
{% if ofsted_relation is not none %}
left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn
{% endif %}
{% if listed_relation is not none %}
left join {{ ref('int_ofsted_listed') }} l on s.urn = l.urn
{% endif %}
-- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when
-- GIAS flips to Closed — marts fully rebuild each run).
where s.status_code in (1, 3)
@@ -14,6 +14,8 @@ unit_tests:
- input: ref('int_ofsted_latest')
rows:
- {urn: 102408, inspection_date: '2025-06-17', overall_effectiveness: null, ungraded_grade: 2, framework: 'Schools - S5', latest_visit_date: '2025-06-17', current_grade: null}
- input: ref('int_ofsted_listed')
rows: []
expect:
rows:
- {urn: 102408, ofsted_grade: null, ofsted_date: '2025-06-17', ofsted_framework: 'Schools - S5'}
@@ -30,6 +32,30 @@ unit_tests:
- input: ref('int_ofsted_latest')
rows:
- {urn: 139888, inspection_date: '2020-03-03', overall_effectiveness: 2, ungraded_grade: null, framework: 'Schools - S5', latest_visit_date: '2025-05-21', current_grade: 2}
- input: ref('int_ofsted_listed')
rows: []
expect:
rows:
- {urn: 139888, ofsted_grade: 2, ofsted_date: '2025-05-21'}
- name: dim_school_carries_inspectorate_and_ofsted_listing
description: >
Audit H4. GIAS names ISI for Abbey College Cambridge; Ofsted lists Abbas
and Templecombe without an inspection; a third school is in neither.
model: dim_school
given:
- input: ref('stg_gias_establishments')
rows:
- {urn: 136083, school_name: 'Abbey College Cambridge', status_code: 1, school_type_code: 11, local_authority_code: 873, phase_code: 4, inspectorate_name: 'ISI'}
- {urn: 150031, school_name: 'Abbas and Templecombe Church School', status_code: 1, school_type_code: 28, local_authority_code: 933, phase_code: 2, inspectorate_name: 'Not applicable'}
- {urn: 1, school_name: 'No data school', status_code: 1, school_type_code: 1, local_authority_code: 312, phase_code: 2, inspectorate_name: null}
- input: ref('int_ofsted_latest')
rows: []
- input: ref('int_ofsted_listed')
rows:
- {urn: 150031}
expect:
rows:
- {urn: 136083, inspectorate: 'ISI', ofsted_listed: false}
- {urn: 150031, inspectorate: 'Not applicable', ofsted_listed: true}
- {urn: 1, inspectorate: null, ofsted_listed: false}
@@ -24,6 +24,12 @@ sources:
- name: ofsted_inspections
description: Ofsted Management Information inspection records
- name: ofsted_fe_providers
description: Ofsted further education and skills MI, newest "as at" file — one row per provider with its latest inspection under the previous framework
- name: ofsted_fe_inspections
description: Ofsted further education and skills MI — renewed-framework inspections (report cards) since 10 November 2025, one row per inspection
- name: ees_ks2_attainment
description: KS2 school attainment (long format — one row per school × subject × breakdown)
@@ -38,6 +38,7 @@ renamed as (
"UrbanRural (name)" as urban_rural,
"ParliamentaryConstituency (name)" as parliamentary_constituency,
"NurseryProvision (name)" as nursery_provision,
nullif(trim("InspectorateName (name)"), '') as inspectorate_name,
cast(nullif("Easting", '') as integer) as easting,
cast(nullif("Northing", '') as integer) as northing,
cast(nullif("StatutoryLowAge", '') as integer) as statutory_low_age,
@@ -0,0 +1,15 @@
-- URN → UKPRN from the raw GIAS extract, for matching Ofsted's further
-- education providers (int_ofsted_fe_latest, int_ofsted_listed; audit H4).
-- A model on the raw source rather than a column of stg_gias_establishments:
-- a ref to that model would put the Ofsted models into the daily build's
-- stg_gias_establishments+ selection (pipeline/tests/test_dag_selectors.py).
-- Built by the monthly Ofsted DAG with the views on it, never by the daily
-- one: rebuilding a view drops the views on it (CASCADE), and dim_school, which
-- the daily build makes, reads int_ofsted_latest and int_ofsted_listed.
select
cast("URN" as integer) as urn,
nullif(trim("UKPRN"), '') as ukprn
from {{ source('raw', 'gias_establishments') }}
-- target-postgres types URN as bigint (the tap declares an integer).
where "URN" is not null
@@ -0,0 +1,29 @@
version: 2
unit_tests:
- name: stg_ofsted_fe_providers_parses_grades_and_dates
description: Ofsted writes "-" for no grade or date; grades are 1-4.
model: stg_ofsted_fe_providers
given:
- input: source('raw', 'ofsted_fe_providers')
rows:
- {provider_urn: '145175', provider_ukprn: '10005429', provider_name: "Queen Mary's College", renewed_eif_full_inspection: 'No', graded_inspection_date: '06/12/2022', overall_effectiveness: '2', short_inspection_date: '-'}
- {provider_urn: '130645', provider_ukprn: '-', provider_name: 'Exeter and North Devon Colleges Group', renewed_eif_full_inspection: 'No', graded_inspection_date: '-', overall_effectiveness: '-', short_inspection_date: '-'}
- {provider_urn: '', provider_ukprn: '10000001', provider_name: 'No URN'}
expect:
rows:
- {provider_urn: 145175, provider_ukprn: '10005429', renewed_eif_full_inspection: false, graded_inspection_date: '2022-12-06', overall_effectiveness: 2, short_inspection_date: null}
- {provider_urn: 130645, provider_ukprn: null, renewed_eif_full_inspection: false, graded_inspection_date: null, overall_effectiveness: null, short_inspection_date: null}
- name: stg_ofsted_fe_inspections_parses_dates_and_safeguarding
model: stg_ofsted_fe_inspections
given:
- input: source('raw', 'ofsted_fe_inspections')
rows:
- {inspection_number: '10380001', provider_urn: '143526', provider_ukprn: '10000002', first_day_of_inspection: '12/05/2026', date_published: '16/06/2026', safeguarding_standards: 'Met'}
- {inspection_number: '10380002', provider_urn: '150042', provider_ukprn: '10000003', first_day_of_inspection: '20/01/2026', date_published: '24/02/2026', safeguarding_standards: 'Not met'}
- {inspection_number: '10380003', provider_urn: '1', provider_ukprn: '', first_day_of_inspection: '', date_published: '', safeguarding_standards: ''}
expect:
rows:
- {inspection_number: '10380001', provider_urn: 143526, rc_inspection_date: '2026-05-12', published_date: '2026-06-16', rc_safeguarding_met: true}
- {inspection_number: '10380002', provider_urn: 150042, rc_inspection_date: '2026-01-20', published_date: '2026-02-24', rc_safeguarding_met: false}
@@ -0,0 +1,18 @@
-- Staging model: Ofsted's renewed-framework FE inspections (report cards),
-- one row per inspection (audit H4). Only the headline is kept: the date and
-- the safeguarding result.
select
trim(inspection_number) as inspection_number,
cast(trim(provider_urn) as integer) as provider_urn,
nullif(nullif(trim(provider_ukprn), ''), '-') as provider_ukprn,
to_date(trim(first_day_of_inspection), 'DD/MM/YYYY') as rc_inspection_date,
case when trim(date_published) ~ '^\d{2}/\d{2}/\d{4}$'
then to_date(trim(date_published), 'DD/MM/YYYY') end as published_date,
case lower(trim(safeguarding_standards))
when 'met' then true
when 'not met' then false
end as rc_safeguarding_met
from {{ source('raw', 'ofsted_fe_inspections') }}
where trim(provider_urn) ~ '^[0-9]+$'
and trim(first_day_of_inspection) ~ '^\d{2}/\d{2}/\d{4}$'
@@ -0,0 +1,18 @@
-- Staging model: Ofsted further education and skills providers — the "as at"
-- MI file, one row per provider with its latest inspection under the previous
-- framework (audit H4). Ofsted writes '-' for no grade or date.
select
cast(trim(provider_urn) as integer) as provider_urn,
nullif(nullif(trim(provider_ukprn), ''), '-') as provider_ukprn,
provider_name,
provider_type,
lower(trim(renewed_eif_full_inspection)) = 'yes' as renewed_eif_full_inspection,
case when trim(graded_inspection_date) ~ '^\d{2}/\d{2}/\d{4}$'
then to_date(trim(graded_inspection_date), 'DD/MM/YYYY') end as graded_inspection_date,
case when trim(overall_effectiveness) in ('1', '2', '3', '4')
then trim(overall_effectiveness)::integer end as overall_effectiveness,
case when trim(short_inspection_date) ~ '^\d{2}/\d{2}/\d{4}$'
then to_date(trim(short_inspection_date), 'DD/MM/YYYY') end as short_inspection_date
from {{ source('raw', 'ofsted_fe_providers') }}
where trim(provider_urn) ~ '^[0-9]+$'
@@ -0,0 +1,7 @@
-- A school-framework record always wins over a further education one for the
-- same URN (audit H4): no FE row may stand where Ofsted's school MI has one.
select f.urn
from {{ ref('fact_ofsted_latest') }} f
join {{ ref('stg_ofsted_inspections') }} s on s.urn = f.urn
where f.framework = 'Further education and skills'