From 88c653215d520ab6e902c9de55bf27d86eeb90c3 Mon Sep 17 00:00:00 2001 From: Tudor Date: Sat, 15 Aug 2026 22:48:30 +0100 Subject: [PATCH] feat(admissions): show the last distance offered where councils publish it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the cut-off distance a parent actually asks about — "how close do we need to live?" — end to end: a Singer tap, dbt staging and mart models, an Airflow DAG, and a tile on both detail templates. 3,597 schools across 57 local authorities carry a figure; the rest are unchanged. There is no national source for this. Each LA publishes its own cut-offs in its own format, and the collected CSV is transcribed from PDFs, spreadsheets and web pages — so most of the work here is deciding what is safe to show. Data * tap-uk-school-distance loads the CSV verbatim into raw. Keyed on (urn, year, school_name), because school_name carries the admission route: (urn, year) alone collides on 118 keys and a reload would have silently dropped every band but one. * stg_school_distance applies a 25 m – 25 km plausibility band. The source contains 0.0-mile rows (published where a school filled on a higher criterion), 1-metre cut-offs, and one reading 533 miles — ~4% of rows, all of which would put a visibly wrong number on a live page. * fact_admission_distance collapses routes to one row per school per year using the furthest, and keeps route_count so the page can say the figure is the widest of several bands rather than the one for a given child. Serving * Kept out of fact_admissions: that mart is EES-derived and near-complete for England, this one covers 57 LAs, and the two refresh independently. * Latest year only. Coverage is ragged — a school may have 2021 and 2026 and nothing between — so a history array would invite a trend line drawn through gaps that are absences of publication, not of a cut-off. * The Admissions section now renders on either source. 3% of the schools that render have a cut-off and no EES admissions row, and gating on admissions alone would have hidden the figure on those pages. Interface * The year travels with the figure everywhere it appears; a cut-off detached from its admissions round is not a fact about anything. * "Not a fixed catchment — it moves every year" sits under every instance, because that is the inference a parent will otherwise draw. * Replaces a hardcoded "Historical distance cut-off data is not available for this school" that appeared on every secondary page, including the ones whose council does publish it. The absence is now stated only when it is real, and names the authority that would hold it. The tint costs the muted tokens their AA margin: measured on the composited backdrop (not the computed one, which reports the untinted card), --text-muted falls to 4.09:1 in dark theme. The tile uses --text-secondary instead — 6.50:1 dark, 6.60:1 light. The DAG is manual, like the other annual ones: councils publish on allocation day, each on its own timetable, so there is no date worth scheduling against. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WDvkyqqHABm4bmth2kjAxE --- .gitignore | 3 + backend/app.py | 1 + backend/data_loader.py | 35 +- backend/models.py | 31 + backend/tests/test_supplementary_batch.py | 30 +- e2e/tests/journeys.spec.ts | 97 +++ mockups/last-distance-offered.html | 591 ++++++++++++++++++ .../components/lastDistanceOffered.test.tsx | 90 +++ .../__tests__/lib/lastDistanceOffered.test.ts | 80 +++ .../__tests__/lib/schoolSections.test.ts | 34 +- .../__tests__/support/renderSchoolDetail.tsx | 2 + nextjs-app/app/school/[slug]/page.tsx | 5 +- .../components/school/AdmissionsSection.tsx | 84 ++- .../school/PrimarySchoolSections.tsx | 12 +- .../school/SecondaryAdmissionsSection.tsx | 52 +- .../school/SecondarySchoolSections.tsx | 10 +- .../components/school/lastDistanceOffered.ts | 61 ++ .../school/schoolSections.module.css | 102 +++ nextjs-app/lib/schoolSections.ts | 13 +- nextjs-app/lib/types.ts | 26 + nextjs-app/lib/utils.ts | 48 ++ pipeline/dags/school_data_pipeline.py | 65 ++ pipeline/meltano.yml | 15 + .../tap-uk-school-distance/pyproject.toml | 16 + .../tap_uk_school_distance/__init__.py | 1 + .../tap_uk_school_distance/tap.py | 135 ++++ .../transform/models/marts/_marts_schema.yml | 16 + .../models/marts/fact_admission_distance.sql | 50 ++ .../transform/models/staging/_stg_sources.yml | 8 + .../models/staging/stg_school_distance.sql | 55 ++ .../assert_admission_distance_plausible.sql | 27 + 31 files changed, 1753 insertions(+), 42 deletions(-) create mode 100644 mockups/last-distance-offered.html create mode 100644 nextjs-app/__tests__/components/lastDistanceOffered.test.tsx create mode 100644 nextjs-app/__tests__/lib/lastDistanceOffered.test.ts create mode 100644 nextjs-app/components/school/lastDistanceOffered.ts create mode 100644 pipeline/plugins/extractors/tap-uk-school-distance/pyproject.toml create mode 100644 pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/__init__.py create mode 100644 pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/tap.py create mode 100644 pipeline/transform/models/marts/fact_admission_distance.sql create mode 100644 pipeline/transform/models/staging/stg_school_distance.sql create mode 100644 pipeline/transform/tests/assert_admission_distance_plausible.sql diff --git a/.gitignore b/.gitignore index 19c8157..4f969df 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ __pycache__/ pipeline/transform/target/ pipeline/transform/logs/ pipeline/transform/.user.yml + +# Playwright MCP scratch output (screenshots, console logs, page snapshots) +.playwright-mcp/ diff --git a/backend/app.py b/backend/app.py index 120fffc..20d9570 100644 --- a/backend/app.py +++ b/backend/app.py @@ -632,6 +632,7 @@ async def get_school_details(request: Request, urn: int): "census": supplementary.get("census"), "admissions": supplementary.get("admissions"), "admissions_history": supplementary.get("admissions_history") or [], + "admission_distance": supplementary.get("admission_distance"), "sen_detail": supplementary.get("sen_detail"), "phonics": supplementary.get("phonics"), "deprivation": supplementary.get("deprivation"), diff --git a/backend/data_loader.py b/backend/data_loader.py index e3aec2f..0640b76 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -18,7 +18,7 @@ from .config import settings from .database import SessionLocal, engine from .models import ( DimSchool, DimLocation, KS2Performance, - FactOfstedInspection, FactAdmissions, + FactOfstedInspection, FactAdmissions, FactAdmissionDistance, FactDeprivation, FactFinance, FactPupilCharacteristics, ) from .ofsted_codes import ofsted_page_url, report_card_labels @@ -723,6 +723,17 @@ def _admissions_row_dict(a) -> dict: } +def _admission_distance_dict(d) -> dict: + """Serialize one fact_admission_distance row for API responses.""" + return { + "year": d.year, + "distance_m": d.distance_m, + "route_count": d.route_count, + "la_name": d.la_name, + "distance_unit_raw": d.distance_unit_raw, + } + + def _census_dict(pc) -> dict: return { "year": pc.year, @@ -759,6 +770,7 @@ def _empty_supplementary() -> dict: "census": None, "admissions": None, "admissions_history": [], + "admission_distance": None, "sen_detail": None, "phonics": None, "deprivation": None, @@ -837,6 +849,27 @@ def get_supplementary_data_batch(db: Session, urns: list[int]) -> dict: result[urn]["admissions"] = rows_for_urn[-1] if rows_for_urn else None _safe(_admissions) + # Last distance offered — latest year per URN. + # + # Only the most recent year is served. Coverage is ragged (a school may + # have 2021 and 2026 but nothing between), so a history array would invite + # a trend line drawn through gaps that are absences of publication rather + # than absences of a cut-off. + def _admission_distance(): + rows = ( + db.query(FactAdmissionDistance) + .filter(FactAdmissionDistance.urn.in_(urns)) + .order_by(FactAdmissionDistance.urn, FactAdmissionDistance.year.desc()) + .all() + ) + seen = set() + for d in rows: + if d.urn in seen: + continue + seen.add(d.urn) + result[d.urn]["admission_distance"] = _admission_distance_dict(d) + _safe(_admission_distance) + # Deprivation — one row per URN. def _deprivation(): rows = ( diff --git a/backend/models.py b/backend/models.py index 22e9b06..b68b342 100644 --- a/backend/models.py +++ b/backend/models.py @@ -188,6 +188,37 @@ class FactAdmissions(Base): admissions_policy = Column(String(100)) +class FactAdmissionDistance(Base): + """Last distance offered — one row per URN per year. + + Separate from FactAdmissions because the source is separate: EES publishes + admissions for the whole country, whereas cut-off distances exist only for + the local authorities that choose to publish them (57 at the time of + writing), and the two refresh on unrelated timetables. + """ + __tablename__ = "fact_admission_distance" + __table_args__ = ( + Index("ix_admission_distance_urn_year", "urn", "year"), + MARTS, + ) + + urn = Column(Integer, primary_key=True) + year = Column(Integer, primary_key=True) + # Straight-line distance in metres from the school to the last home offered + # a place that year. + distance_m = Column(Float) + # How many admission routes (ability bands, separate reception/junior + # intakes) were collapsed into distance_m. >1 means the figure is the + # furthest of several and the page must say so. + route_count = Column(Integer) + la_code = Column(Integer) + la_name = Column(String(100)) + # The unit the council published in, so the page can lead with the unit a + # parent was given rather than always converting. + distance_unit_raw = Column(String(20)) + source_file = Column(Text) + + class FactPupilCharacteristics(Base): """School pupil composition from EES census — one row per URN per year.""" __tablename__ = "fact_pupil_characteristics" diff --git a/backend/tests/test_supplementary_batch.py b/backend/tests/test_supplementary_batch.py index 0593034..2bdbdcd 100644 --- a/backend/tests/test_supplementary_batch.py +++ b/backend/tests/test_supplementary_batch.py @@ -69,6 +69,13 @@ def _adm_row(urn, year): ) +def _dist_row(urn, year, distance_m, route_count=1): + return types.SimpleNamespace( + urn=urn, year=year, distance_m=distance_m, route_count=route_count, + la_name="Camden", distance_unit_raw="miles", source_file="camden/guide.pdf", + ) + + def test_one_query_per_table_and_latest_row_per_urn(): rows = { # URN 1 has two Ofsted rows; the batch must keep the most recent (2023). @@ -78,6 +85,14 @@ def test_one_query_per_table_and_latest_row_per_urn(): _ofsted_row(2, "2021-06-01", 1), ], "FactAdmissions": [_adm_row(1, 202526), _adm_row(1, 202627), _adm_row(2, 202627)], + # URN 1 has three years of cut-offs; only the most recent is served. + # Deliberately not in year order — the ordering is the query's job. + "FactAdmissionDistance": [ + _dist_row(1, 2026, 529.47), + _dist_row(1, 2024, 772.49), + _dist_row(1, 2025, 1421.05), + _dist_row(2, 2023, 2029.38, route_count=4), + ], "FactPupilCharacteristics": [], "FactDeprivation": [], "FactFinance": [], @@ -85,10 +100,10 @@ def test_one_query_per_table_and_latest_row_per_urn(): session = _FakeSession(rows) out = get_supplementary_data_batch(session, [1, 2]) - # Exactly one query per table — five total, regardless of two URNs. + # Exactly one query per table — six total, regardless of two URNs. assert sorted(session.queries) == [ - "FactAdmissions", "FactDeprivation", "FactFinance", - "FactOfstedInspection", "FactPupilCharacteristics", + "FactAdmissionDistance", "FactAdmissions", "FactDeprivation", + "FactFinance", "FactOfstedInspection", "FactPupilCharacteristics", ] # Latest Ofsted kept per URN @@ -100,6 +115,14 @@ def test_one_query_per_table_and_latest_row_per_urn(): assert out[1]["admissions"]["year"] == 202627 assert out[2]["admissions_history"] == [{**out[2]["admissions_history"][0]}] + # Cut-off distance: latest year only, per URN. Coverage is ragged, so + # serving anything but the newest would show a parent a stale figure. + assert out[1]["admission_distance"]["year"] == 2026 + assert out[1]["admission_distance"]["distance_m"] == 529.47 + # route_count travels with the figure — the page needs it to say the + # distance is the furthest of several bands rather than the only one. + assert out[2]["admission_distance"]["route_count"] == 4 + # Empty tables degrade to the null block, not a crash assert out[1]["census"] is None and out[1]["deprivation"] is None @@ -109,3 +132,4 @@ def test_single_wrapper_matches_batch(monkeypatch): single = data_loader.get_supplementary_data(session, 5) assert single["ofsted"]["overall_effectiveness"] == 2 assert single["admissions_history"] == [] + assert single["admission_distance"] is None diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index a1d3392..07c59a8 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -1210,3 +1210,100 @@ for (const scheme of ['light', 'dark'] as const) { expect(failures, `AA failures in ${scheme}:\n ${failures.join('\n ')}`).toEqual([]); }); } + +// ── Last distance offered ──────────────────────────────────────────────── +// +// Coverage is partial by nature: only 57 local authorities publish cut-off +// distances at all, and staging may not yet hold the extract. These tests +// therefore locate a school that HAS a figure and assert the invariants that +// make it safe to show, rather than asserting any particular school has one. + +/** URNs known to be in the collected cut-off dataset, spread across LAs so a + * single council's data gap cannot skip the whole check. */ +const CUTOFF_CANDIDATE_URNS = [ + 139837, 100010, 100011, 100028, 100029, 130342, // Camden + 100458, 135587, // Islington (banded) + 101099, 100553, 102574, 100769, // mixed +]; + +async function schoolWithCutoff(page: Page) { + for (const urn of CUTOFF_CANDIDATE_URNS) { + const res = await page.request.get(`/api/schools/${urn}`); + if (!res.ok()) continue; + const body = await res.json(); + if (body?.admission_distance?.distance_m != null) { + return { urn, distance: body.admission_distance, phase: body.school_info?.phase }; + } + } + return null; +} + +test('a published cut-off distance is shown with the year it belongs to', async ({ page }) => { + const found = await schoolWithCutoff(page); + test.skip(found === null, 'no school in the sample has a published cut-off distance yet'); + + await page.goto(`/school/${found!.urn}`); + await expect(page.locator('h1').first()).toBeVisible({ timeout: 15_000 }); + + const label = page.getByText(/Last distance offered/).first(); + await expect(label).toBeVisible(); + + // The year is the difference between a fact and a floating number: a cut-off + // is the outcome of one admissions round and means nothing without it. + await expect(label).toContainText(`September ${found!.distance.year}`); + + // And the figure itself, in the unit councils publish in. + await expect(page.getByText(/\d+(\.\d+)? (miles|m)\b/).first()).toBeVisible(); +}); + +test('a cut-off distance is never shown without saying it is not a catchment', async ({ page }) => { + const found = await schoolWithCutoff(page); + test.skip(found === null, 'no school in the sample has a published cut-off distance yet'); + + await page.goto(`/school/${found!.urn}`); + await expect(page.locator('h1').first()).toBeVisible({ timeout: 15_000 }); + + await expect(page.getByText(/not a fixed catchment/)).toBeVisible(); + + // The old secondary template asserted "distance cut-off data is not + // available for this school" on every page, including ones whose council + // does publish it. Showing both at once would be a flat contradiction. + await expect(page.getByText(/has not published a cut-off distance/)).toHaveCount(0); + + // A banded school's figure is the widest of several routes, and must say so. + if ((found!.distance.route_count ?? 1) > 1) { + await expect(page.getByText(/admission routes at this school/)).toBeVisible(); + } +}); + +test('a cut-off distance opens the Admissions section in the sticky nav', async ({ page }) => { + // buildNavItems and the section composers gate on the same condition; when + // they drift the nav links to an anchor that was never rendered. + const found = await schoolWithCutoff(page); + test.skip(found === null, 'no school in the sample has a published cut-off distance yet'); + + await page.goto(`/school/${found!.urn}`); + await expect(page.locator('h1').first()).toBeVisible({ timeout: 15_000 }); + + await expect(page.locator('#admissions')).toHaveCount(1); +}); + +test('no school page shows an implausible cut-off distance', async ({ page }) => { + // The source carries transcription errors up to 533 miles, filtered in + // stg_school_distance. This asserts the filter is actually reaching the API, + // because a visibly absurd figure on a live page is the failure that would + // cost the site its credibility. + let checked = 0; + for (const urn of CUTOFF_CANDIDATE_URNS) { + const res = await page.request.get(`/api/schools/${urn}`); + if (!res.ok()) continue; + const d = (await res.json())?.admission_distance; + if (d?.distance_m == null) continue; + checked += 1; + expect(d.distance_m, `URN ${urn} cut-off outside the plausibility band`) + .toBeGreaterThanOrEqual(25); + expect(d.distance_m, `URN ${urn} cut-off outside the plausibility band`) + .toBeLessThanOrEqual(25000); + } + test.skip(checked === 0, 'no cut-off distances available to check yet'); +}); diff --git a/mockups/last-distance-offered.html b/mockups/last-distance-offered.html new file mode 100644 index 0000000..a182865 --- /dev/null +++ b/mockups/last-distance-offered.html @@ -0,0 +1,591 @@ +Last distance offered — detail page mockup + + +
+ +
+

Last distance offered — school detail page

+

Adds the final-offer cut-off distance to the existing Admissions card, plus a catchment + view on the map. Data covers one to ten years depending on the school and local authority, so every + screen here is built around partial coverage rather than assuming a full run.

+

Sample school: Fairlawn Primary School, Lewisham — 8 years of distance data out of 10 years of admissions data.

+
+ + +
+

1. A distance tile joins the admissions tiles

+ SchoolDetailView · #admissions +
+

No new section and no new nav entry — the number a parent actually asks for + ("how close do we need to live?") sits with the rest of the intake story. The segmented control gains a + third view, Distance.

+ +
+
+

Admissions

+
+ + + +
+
+

Reception entry, September 2025.

+ +
+ +
+
+
+
60
Places offered
+
+
+
142
Wanted it first
+
+
+
54of 142 · 38%
+
Got their first choice
+
+
+ New +
0.31mi≈ 500 m · 6 min walk
+
Last distance offered
+
+
+
+ + The furthest home offered a place once siblings, faith and EHCP priority were applied. + It is not a fixed catchment — it moves every year with the number of applications. +
+
+ + + +
+
+ + +
+

2. The cut-off drawn on the map

+ SchoolHeroMap · catchment layer +
+

"0.31 miles" is abstract until you see it over your own streets. The latest year is a + filled ring; earlier years sit behind it as hairlines, so the tightening reads instantly as a set of + shrinking circles. Straight-line rings only — they're an illustration of the number, not a boundary.

+ +
+
+

Where the last place went

+
+ + +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2016 · 0.62 mi + + + 2025 · 0.31 mi + + + + Fairlawn Primary + + + + + + Your home · 0.24 mi + +
+ +
+ 2025 cut-off — 0.31 mi + Earlier years, 2016–2024 + Your home +
+ +
+
Would you have got in?
+

We measure straight-line distance from your postcode, the same way Lewisham does.

+
+ + + +
+ +
+
0.24 miles away — inside the cut-off in all 8 years on record
+
That's 0.07 miles of headroom on 2025, the tightest year so far. 2018 has no published + figure, and in 2022 every applicant was offered a place.
+
+ 2016 ✓ + 2017 ✓ + 2018 – + 2019 ✓ + 2020 ✓ + 2021 ✓ + 2022 ✓ + 2023 ✓ + 2024 ✓ + 2025 ✓ +
+
+ +
+ + An indication only. Distance is applied after siblings, faith and EHCP priority, and next year's + cut-off depends on next year's applicants. Always check the school's own admissions policy. +
+
+
+ + +
+

3. Coverage states

+ Partial data is the normal case +
+

Coverage runs from ten years to none. Each state says something true rather than + falling back on a generic "no data" — the reason a figure is absent is itself useful to a parent.

+ +
+
+

4+ years

+
Full treatment: verdict banner, chart, table, map rings. The verdict line only + appears with 4+ points — below that a two-year swing isn't a trend.
+
+
+

2–3 years

+
Tile and table, no verdict banner, single map ring for the latest year. + "Only 3 years available" sits under the table.
+
+
+

1 year

+
Tile plus one map ring. No Distance tab — the tile and its footnote are + the whole story.
+
+
+

Never oversubscribed

+
Not missing data — good news, and it should read that way.
+
Everyone who applied was offered a place + in each of the last 6 years, so no distance cut-off was needed.
+
+
+

No data at all

+
Replaces the current placeholder text in + SecondarySchoolDetailView.tsx:780.
+
Lewisham hasn't published cut-off distances for this school. + The council's admissions page may list them.
+
+
+

Not distance-ranked

+
Grammar and some faith schools rank on test score or faith + practice, so a distance figure would mislead.
+
Places here are ranked by the entrance test, not by distance.
+
+
+ + +
+

4. Mobile

+ ≤ 640 px +
+

Tiles fall to two columns and the distance tile takes the full width beneath them, so the + headline number survives the reflow. The chart drops its table on mobile behind a "See all years" disclosure.

+ +
+
+
+
+
+

Admissions

+
+

Reception, Sept 2025

+
+ + + +
+
+
60
Places offered
+
142
Wanted it first
+
+
+ New +
0.31mi + ≈ 500 m · 6 min walk
+
Last distance offered, 2025
+
+
+ +
+

Distance

+
+ +
Halved in nine years
+
0.62 mi → 0.31 mi
+
+
+ + + + + + 0.80.50.2 + + + + + + + + + + + + + + '16'20'25 + + 0.31 mi + +
+
+ See all 10 years +
+
+
+
+ +
+

Mobile decisions

+
    +
  • Distance tile spans both columns — the number a parent came for shouldn't be a half-width cell.
    • +
  • Chart keeps every year but labels only first, middle and last; the endpoint stays labelled.
    • +
  • Year-by-year table collapses into a disclosure rather than forcing a horizontal scroll.
    • +
  • Map rings reuse the existing full-screen hero map sheet, opened from the tile.
    • +
+
+
+ +
+ + diff --git a/nextjs-app/__tests__/components/lastDistanceOffered.test.tsx b/nextjs-app/__tests__/components/lastDistanceOffered.test.tsx new file mode 100644 index 0000000..7472c5f --- /dev/null +++ b/nextjs-app/__tests__/components/lastDistanceOffered.test.tsx @@ -0,0 +1,90 @@ +/** + * Last distance offered, rendered on both detail templates. + * + * The figure is the one number on these pages that a parent may act on — it is + * easy to read as "we live inside the catchment, we will get a place". These + * tests pin the things that stop it being read that way: the year is always + * present, the caveat is always present, and the blanket "not available" + * sentence appears only when it is actually true. + */ + +import { screen } from '@testing-library/react'; +import { renderSchoolDetail, renderSecondarySchoolDetail } from '../support/renderSchoolDetail'; +import { primaryFixture, secondaryFixture } from '../support/schoolFixtures'; +import type { SchoolAdmissionDistance } from '@/lib/types'; + +const cutoff = (over: Partial = {}): SchoolAdmissionDistance => ({ + year: 2025, + distance_m: 500, + route_count: 1, + la_name: 'Camden', + distance_unit_raw: 'miles', + ...over, +}); + +describe('primary detail page', () => { + it('shows the figure with the year it belongs to', () => { + renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff({ distance_m: 772.49, year: 2024 }) }); + + expect(screen.getByText('0.48 miles')).toBeInTheDocument(); + expect(screen.getByText(/Last distance offered/)).toHaveTextContent('September 2024'); + }); + + it('never shows the figure without saying it is not a catchment', () => { + renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff() }); + + expect(screen.getByText(/not a fixed catchment/)).toBeInTheDocument(); + expect(screen.getByText(/moves every year/)).toBeInTheDocument(); + }); + + it('flags that a banded school\'s figure is the widest of several routes', () => { + renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff({ route_count: 4 }) }); + + expect(screen.getByText(/4 admission routes/)).toBeInTheDocument(); + }); + + it('renders nothing distance-related when the LA publishes none', () => { + renderSchoolDetail({ ...primaryFixture, admissionDistance: null }); + + expect(screen.queryByText(/Last distance offered/)).not.toBeInTheDocument(); + expect(screen.queryByText(/not a fixed catchment/)).not.toBeInTheDocument(); + }); + + it('carries the figure even with no EES admissions row', () => { + // The two sources are independent; this school has a cut-off and no + // admissions figures. Before this feature the section did not render at all. + renderSchoolDetail({ + ...primaryFixture, + admissions: null, + admissionsHistory: [], + admissionDistance: cutoff({ distance_m: 1421.05 }), + }); + + expect(screen.getByText('0.88 miles')).toBeInTheDocument(); + }); +}); + +describe('secondary detail page', () => { + it('shows the figure with the year it belongs to', () => { + renderSecondarySchoolDetail({ ...secondaryFixture, admissionDistance: cutoff({ distance_m: 3472.96 }) }); + + expect(screen.getByText('2.16 miles')).toBeInTheDocument(); + expect(screen.getByText(/Last distance offered/)).toHaveTextContent('September 2025'); + }); + + it('drops the blanket "not available" line once a distance exists', () => { + renderSecondarySchoolDetail({ ...secondaryFixture, admissionDistance: cutoff() }); + + expect(screen.queryByText(/has not published a cut-off distance/)).not.toBeInTheDocument(); + expect(screen.getByText(/not a fixed catchment/)).toBeInTheDocument(); + }); + + it('names the authority that would hold the data when there is none', () => { + // The old copy asserted "Historical distance cut-off data is not available + // for this school" on every secondary page, including the ones whose + // council does publish it. + renderSecondarySchoolDetail({ ...secondaryFixture, admissionDistance: null }); + + expect(screen.getByText(/has not published a cut-off distance/)).toBeInTheDocument(); + }); +}); diff --git a/nextjs-app/__tests__/lib/lastDistanceOffered.test.ts b/nextjs-app/__tests__/lib/lastDistanceOffered.test.ts new file mode 100644 index 0000000..0de75d0 --- /dev/null +++ b/nextjs-app/__tests__/lib/lastDistanceOffered.test.ts @@ -0,0 +1,80 @@ +/** + * Last distance offered — formatting and the caveats attached to the figure. + * + * The assertions about the route note and the year are not cosmetic. A cut-off + * shown without its year, or a banded school's widest cut-off shown as if it + * were the only one, tells a parent something false about their chances of a + * place — so both are pinned here rather than left to the component. + */ + +import { formatCutoffDistance, formatEntryYear } from '@/lib/utils'; +import { describeCutoff } from '@/components/school/lastDistanceOffered'; +import type { SchoolAdmissionDistance } from '@/lib/types'; + +const distance = (over: Partial = {}): SchoolAdmissionDistance => ({ + year: 2025, + distance_m: 500, + route_count: 1, + la_name: 'Camden', + distance_unit_raw: 'miles', + ...over, +}); + +describe('formatCutoffDistance', () => { + it('leads with miles, the unit councils publish in', () => { + expect(formatCutoffDistance(500)).toEqual({ primary: '0.31 miles', secondary: '500 m' }); + expect(formatCutoffDistance(1609.344)).toEqual({ primary: '1.00 miles', secondary: '1.6 km' }); + }); + + it('switches to kilometres for the support figure above a kilometre', () => { + expect(formatCutoffDistance(3472.96)!.secondary).toBe('3.5 km'); + }); + + it('leads with metres below 100 m, where a miles figure carries nothing', () => { + expect(formatCutoffDistance(27)).toEqual({ primary: '27 m', secondary: '0.02 miles' }); + }); + + it('returns null rather than a zero cut-off', () => { + // 0.0 miles appears in the source where a school filled on a higher + // criterion. Rendered as "0.00 miles" it would read as the opposite. + expect(formatCutoffDistance(0)).toBeNull(); + expect(formatCutoffDistance(null)).toBeNull(); + expect(formatCutoffDistance(undefined)).toBeNull(); + expect(formatCutoffDistance(Number.NaN)).toBeNull(); + }); +}); + +describe('formatEntryYear', () => { + it('names the intake, not the academic year', () => { + // formatAcademicYear would render 2025 as "2025/26", which reads as a + // school year rather than the September a child started. + expect(formatEntryYear(2025)).toBe('September 2025'); + expect(formatEntryYear(null)).toBe(''); + }); +}); + +describe('describeCutoff', () => { + it('always carries the entry year alongside the figure', () => { + const d = describeCutoff(distance({ distance_m: 772.49, year: 2024 })); + expect(d).not.toBeNull(); + expect(d!.primary).toBe('0.48 miles'); + expect(d!.entryYear).toBe('September 2024'); + }); + + it('says nothing about routes for a school with one', () => { + expect(describeCutoff(distance({ route_count: 1 }))!.routeNote).toBeNull(); + expect(describeCutoff(distance({ route_count: null }))!.routeNote).toBeNull(); + }); + + it('warns that a banded school\'s figure is the widest of several', () => { + const note = describeCutoff(distance({ route_count: 4 }))!.routeNote; + expect(note).toContain('4 admission routes'); + expect(note).toContain('shorter cut-off'); + }); + + it('is null when there is nothing publishable', () => { + expect(describeCutoff(null)).toBeNull(); + expect(describeCutoff(undefined)).toBeNull(); + expect(describeCutoff(distance({ distance_m: null }))).toBeNull(); + }); +}); diff --git a/nextjs-app/__tests__/lib/schoolSections.test.ts b/nextjs-app/__tests__/lib/schoolSections.test.ts index 0c893d7..acaec4e 100644 --- a/nextjs-app/__tests__/lib/schoolSections.test.ts +++ b/nextjs-app/__tests__/lib/schoolSections.test.ts @@ -60,7 +60,7 @@ describe('buildNavItems', () => { it('omits sections with no data', () => { const flags = computeSchoolFlags(specialFixture); const ids = buildNavItems(flags, { - ofsted: null, admissions: null, yearlyDataLength: 1, + ofsted: null, admissions: null, admissionDistance: null, yearlyDataLength: 1, }).map((n) => n.id); expect(ids).not.toContain('ofsted'); @@ -74,6 +74,7 @@ describe('buildNavItems', () => { return buildNavItems(flags, { ofsted: fixture.ofsted, admissions: fixture.admissions, + admissionDistance: null, yearlyDataLength: fixture.yearlyData.length, }).find((n) => n.id === 'results')?.label; }; @@ -83,11 +84,27 @@ describe('buildNavItems', () => { expect(label(allThroughFixture)).toBe('Results'); }); + it('opens the admissions entry for a cut-off distance with no EES admissions', () => { + // 3% of the schools that render have one source and not the other. The nav + // condition and the section's render condition have to agree, or the sticky + // nav links to an anchor that was never rendered. + const flags = computeSchoolFlags(specialFixture); + const ids = buildNavItems(flags, { + ofsted: null, + admissions: null, + admissionDistance: { year: 2025, distance_m: 500, route_count: 1, la_name: 'Camden', distance_unit_raw: 'miles' }, + yearlyDataLength: 1, + }).map((n) => n.id); + + expect(ids).toContain('admissions'); + }); + it('keeps the engagement-led ordering', () => { const flags = computeSchoolFlags(primaryFixture); const ids = buildNavItems(flags, { ofsted: primaryFixture.ofsted, admissions: primaryFixture.admissions, + admissionDistance: null, yearlyDataLength: primaryFixture.yearlyData.length, }).map((n) => n.id); @@ -127,16 +144,29 @@ describe('buildSecondaryNavItems', () => { const ids = buildSecondaryNavItems(flags, { ofsted: secondaryFixture.ofsted, admissions: secondaryFixture.admissions, + admissionDistance: null, yearlyDataLength: secondaryFixture.yearlyData.length, }).map((n) => n.id); expect(ids).toEqual(['ofsted', 'gcse', 'admissions', 'history', 'wellbeing', 'finances']); }); + it('opens the admissions entry for a cut-off distance alone', () => { + const flags = computeSecondaryFlags(secondaryFixture); + const ids = buildSecondaryNavItems(flags, { + ofsted: null, + admissions: null, + admissionDistance: { year: 2025, distance_m: 2400, route_count: 4, la_name: 'Islington', distance_unit_raw: 'miles' }, + yearlyDataLength: 1, + }).map((n) => n.id); + + expect(ids).toContain('admissions'); + }); + it('gates History on more than one year, unlike the primary page', () => { const flags = computeSecondaryFlags(secondaryFixture); const ids = buildSecondaryNavItems(flags, { - ofsted: null, admissions: null, yearlyDataLength: 1, + ofsted: null, admissions: null, admissionDistance: null, yearlyDataLength: 1, }).map((n) => n.id); expect(ids).not.toContain('history'); diff --git a/nextjs-app/__tests__/support/renderSchoolDetail.tsx b/nextjs-app/__tests__/support/renderSchoolDetail.tsx index 72deca2..9dc43bd 100644 --- a/nextjs-app/__tests__/support/renderSchoolDetail.tsx +++ b/nextjs-app/__tests__/support/renderSchoolDetail.tsx @@ -31,6 +31,7 @@ export function renderSchoolDetail(fixture: any) { const navItems = buildNavItems(flags, { ofsted: fixture.ofsted, admissions: fixture.admissions, + admissionDistance: fixture.admissionDistance ?? null, yearlyDataLength: fixture.yearlyData.length, }); @@ -57,6 +58,7 @@ export function renderSecondarySchoolDetail(fixture: any) { const navItems = buildSecondaryNavItems(flags, { ofsted: fixture.ofsted, admissions: fixture.admissions, + admissionDistance: fixture.admissionDistance ?? null, yearlyDataLength: fixture.yearlyData.length, }); diff --git a/nextjs-app/app/school/[slug]/page.tsx b/nextjs-app/app/school/[slug]/page.tsx index 538ee3d..55a156d 100644 --- a/nextjs-app/app/school/[slug]/page.tsx +++ b/nextjs-app/app/school/[slug]/page.tsx @@ -147,7 +147,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) { notFound(); } - const { school_info, yearly_data, absence_data, ofsted, census, admissions, admissions_history, deprivation, finance } = data; + const { school_info, yearly_data, absence_data, ofsted, census, admissions, admissions_history, admission_distance, deprivation, finance } = data; // Redirect bare URN to canonical slug URL const canonicalSlug = schoolUrl(urn, school_info.school_name).replace('/school/', ''); @@ -176,6 +176,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) { const navInput = { ofsted: ofsted ?? null, admissions: admissions ?? null, + admissionDistance: admission_distance ?? null, yearlyDataLength: yearly_data.length, }; const primaryNavItems = buildNavItems(primaryFlags, navInput); @@ -228,6 +229,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) { ofsted={ofsted ?? null} census={census ?? null} admissions={admissions ?? null} + admissionDistance={admission_distance ?? null} deprivation={deprivation ?? null} finance={finance ?? null} nationalAvg={nationalAvg} @@ -249,6 +251,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) { census={census ?? null} admissions={admissions ?? null} admissionsHistory={admissions_history ?? []} + admissionDistance={admission_distance ?? null} deprivation={deprivation ?? null} finance={finance ?? null} nationalAvg={nationalAvg} diff --git a/nextjs-app/components/school/AdmissionsSection.tsx b/nextjs-app/components/school/AdmissionsSection.tsx index def1d3b..4b9f6c8 100644 --- a/nextjs-app/components/school/AdmissionsSection.tsx +++ b/nextjs-app/components/school/AdmissionsSection.tsx @@ -12,59 +12,112 @@ */ import type { ReactNode } from 'react'; -import type { SchoolAdmissions } from '@/lib/types'; +import type { SchoolAdmissions, SchoolAdmissionDistance } from '@/lib/types'; import { formatAcademicYear, formatPercentage } from '@/lib/utils'; import { summariseAdmissions } from '@/lib/compareLogic'; import { Section, sectionStyles as styles } from './sectionShared'; import { AdmissionsViewToggle } from './AdmissionsViewToggle'; import { AdmissionsTrendChart } from './charts'; +import { describeCutoff, CUTOFF_NOTE, CUTOFF_MEASUREMENT_NOTE } from './lastDistanceOffered'; export function AdmissionsSection({ admissions, admissionsHistory, + admissionDistance, isAllThrough, }: { - admissions: SchoolAdmissions; + /* Nullable: the two data sources are independent. A minority of schools have + a published cut-off distance but no EES admissions row (3% of the schools + that render, sampled against staging), and for those this section exists + to carry the distance alone. */ + admissions: SchoolAdmissions | null; admissionsHistory: SchoolAdmissions[]; + admissionDistance: SchoolAdmissionDistance | null; isAllThrough: boolean; }) { + const cutoff = describeCutoff(admissionDistance); + // Trend toggle only appears with ≥2 years carrying an offer rate. const admissionsOfferYears = admissionsHistory.filter((h) => h.first_preference_offer_pct != null).length; - const showAdmissionsTrend = admissionsOfferYears >= 2; + const showAdmissionsTrend = admissions != null && admissionsOfferYears >= 2; // Banded interpretation of the first-choice offer rate ("More than half of // first choices missed out" etc.) — the same banding the compare screen // uses, so a low offer rate reads as how severe it actually is. - const admissionsSummary = summariseAdmissions(admissions); + const admissionsSummary = admissions ? summariseAdmissions(admissions) : null; - const title = <>Admissions{!showAdmissionsTrend && ` (${formatAcademicYear(admissions.year)})`}; + const title = ( + <>Admissions{!showAdmissionsTrend && admissions && ` (${formatAcademicYear(admissions.year)})`} + ); {/* All-through admissions data covers a single entry point (usually the Year 7 secondary intake), not reception — say so, or a parent could read these as the whole-school figures. */} - const subtitle: ReactNode = isAllThrough && admissions.school_phase ? ( + const subtitle: ReactNode = isAllThrough && admissions?.school_phase ? (

These figures are for {admissions.school_phase.toLowerCase()} entry {/secondary/i.test(admissions.school_phase) ? ' (Year 7)' : /primary/i.test(admissions.school_phase) ? ' (Reception)' : ''}.

) : null; + {/* Spans both columns rather than taking a half-width cell. This is the + figure parents come to the page for, and at tile width the two-line + "0.31 miles / September 2025" pairing wraps badly. */} + const distanceTile = cutoff && ( +
+
+ {cutoff.primary} + {cutoff.secondary} +
+
+ Last distance offered · {cutoff.entryYear} +
+
+ ); + + const distanceNote = cutoff && ( +

+ {CUTOFF_NOTE} {CUTOFF_MEASUREMENT_NOTE} + {cutoff.routeNote && <> {cutoff.routeNote}} +

+ ); + + /* + * Which row template the tile grid needs. + * + * The grid reserves two equal rows so the year view matches the height of the + * trend chart beside it. With a cut-off and no EES admissions there are no + * metric tiles to fill them, and the reserved rows render as a bare block of + * the grid's own gap colour under the distance tile. + */ + const hasMetricTiles = admissions != null && ( + admissions.places_offered != null + || admissions.first_preference_applications != null + || admissions.first_preference_offer_pct != null + || admissions.total_applications != null + ); + const tilesClass = [ + styles.admissionsTiles, + cutoff && hasMetricTiles ? styles.admissionsTilesWithDistance : '', + cutoff && !hasMetricTiles ? styles.admissionsTilesDistanceOnly : '', + ].filter(Boolean).join(' '); + const yearView = ( <> -
- {admissions.places_offered != null && ( +
+ {admissions?.places_offered != null && (
{admissions.places_offered}
Places offered
)} - {admissions.first_preference_applications != null && ( + {admissions?.first_preference_applications != null && (
{admissions.first_preference_applications}
Wanted it first
)} - {admissions.first_preference_offer_pct != null && ( + {admissions?.first_preference_offer_pct != null && (
{admissions.first_preference_offers != null && admissions.first_preference_applications != null ? ( @@ -81,20 +134,25 @@ export function AdmissionsSection({
Got their first choice
)} - {admissions.total_applications != null && ( + {admissions?.total_applications != null && (
{admissions.total_applications.toLocaleString()}
Applied in total
)} + {distanceTile}
- {admissionsSummary.chip && ( + {admissionsSummary?.chip && (

{admissionsSummary.chip.text}

)} + {distanceNote} ); - const trendView = ( + {/* Guarded rather than asserted: showAdmissionsTrend already requires + admissions, and tying the two together in one expression keeps that + invariant checked by the compiler instead of assumed. */} + const trendView = admissions && ( <>
First-choice offer rate
diff --git a/nextjs-app/components/school/PrimarySchoolSections.tsx b/nextjs-app/components/school/PrimarySchoolSections.tsx index 2cafb22..b2bff36 100644 --- a/nextjs-app/components/school/PrimarySchoolSections.tsx +++ b/nextjs-app/components/school/PrimarySchoolSections.tsx @@ -13,7 +13,7 @@ import type { School, SchoolResult, AbsenceData, OfstedInspection, SchoolCensus, - SchoolAdmissions, SchoolDeprivation, SchoolFinance, NationalAverages, + SchoolAdmissions, SchoolAdmissionDistance, SchoolDeprivation, SchoolFinance, NationalAverages, } from '@/lib/types'; import { ofstedLegacyAreas } from '@/lib/utils'; import type { SchoolFlags } from '@/lib/schoolSections'; @@ -34,6 +34,7 @@ export interface PrimarySchoolSectionsProps { census: SchoolCensus | null; admissions: SchoolAdmissions | null; admissionsHistory: SchoolAdmissions[]; + admissionDistance: SchoolAdmissionDistance | null; deprivation: SchoolDeprivation | null; finance: SchoolFinance | null; nationalAvg: NationalAverages | null; @@ -42,7 +43,7 @@ export interface PrimarySchoolSectionsProps { export function PrimarySchoolSections({ schoolInfo, yearlyData, absenceData, ofsted, census, - admissions, admissionsHistory, deprivation, finance, nationalAvg, flags, + admissions, admissionsHistory, admissionDistance, deprivation, finance, nationalAvg, flags, }: PrimarySchoolSectionsProps) { const primaryAvg = nationalAvg?.primary ?? {}; const secondaryAvg = nationalAvg?.secondary ?? {}; @@ -91,10 +92,15 @@ export function PrimarySchoolSections({ /> )} - {admissions && ( + {/* Either source is enough to justify the section. The cut-off distance + and the EES admissions figures come from different places and a + minority of schools have one without the other — gating on admissions + alone would hide a published distance on those pages. */} + {(admissions || admissionDistance) && ( )} diff --git a/nextjs-app/components/school/SecondaryAdmissionsSection.tsx b/nextjs-app/components/school/SecondaryAdmissionsSection.tsx index bd0fbf8..7e8fdb3 100644 --- a/nextjs-app/components/school/SecondaryAdmissionsSection.tsx +++ b/nextjs-app/components/school/SecondaryAdmissionsSection.tsx @@ -6,17 +6,22 @@ * JavaScript at all. Server component. */ -import type { School, SchoolAdmissions } from '@/lib/types'; +import type { School, SchoolAdmissions, SchoolAdmissionDistance } from '@/lib/types'; import { formatPercentage } from '@/lib/utils'; import { Section, sectionStyles as styles } from './sectionShared'; +import { describeCutoff, CUTOFF_NOTE, CUTOFF_MEASUREMENT_NOTE } from './lastDistanceOffered'; export function SecondaryAdmissionsSection({ - admissions, schoolInfo, hasSixthForm, + admissions, admissionDistance, schoolInfo, hasSixthForm, }: { - admissions: SchoolAdmissions; + /* Nullable for the same reason as the primary section: a school can have a + published cut-off and no EES admissions row. */ + admissions: SchoolAdmissions | null; + admissionDistance: SchoolAdmissionDistance | null; schoolInfo: School; hasSixthForm: boolean; }) { + const cutoff = describeCutoff(admissionDistance); // Moved with this section from SecondarySchoolDetailView, its only consumer. const admissionsTag = (() => { const policy = schoolInfo.admissions_policy?.toLowerCase() ?? ''; @@ -40,32 +45,43 @@ export function SecondaryAdmissionsSection({ )}
- {admissions.places_offered != null && ( + {admissions?.places_offered != null && (
Year 7 places offered
{admissions.places_offered}
)} - {admissions.total_applications != null && ( + {admissions?.total_applications != null && (
Total applications
{admissions.total_applications.toLocaleString()}
)} - {admissions.first_preference_applications != null && ( + {admissions?.first_preference_applications != null && (
1st preference applications
{admissions.first_preference_applications.toLocaleString()}
)} - {admissions.first_preference_offer_pct != null && ( + {admissions?.first_preference_offer_pct != null && (
Families who got their first choice
{formatPercentage(admissions.first_preference_offer_pct)}
)} + {cutoff && ( +
+
+ Last distance offered · {cutoff.entryYear} +
+
+ {cutoff.primary} + {cutoff.secondary} +
+
+ )}
- {admissions.oversubscribed != null && ( + {admissions?.oversubscribed != null && (
{admissions.oversubscribed ? '⚠ Applications exceeded places last year' @@ -73,9 +89,23 @@ export function SecondaryAdmissionsSection({
)} -

- Historical distance cut-off data is not available for this school. Contact the admissions authority for oversubscription criteria details. -

+ {/* Replaces a blanket "distance cut-off data is not available for this + school", which was hardcoded onto every secondary page and was untrue + wherever the local authority does publish. The absence is now stated + only when it is real, and names the authority that would hold it. */} + {cutoff ? ( +

+ {CUTOFF_NOTE} {CUTOFF_MEASUREMENT_NOTE} + {cutoff.routeNote && <> {cutoff.routeNote}} +

+ ) : ( +

+ {schoolInfo.local_authority + ? `${schoolInfo.local_authority} has not published a cut-off distance for this school.` + : 'No cut-off distance has been published for this school.'}{' '} + Contact the admissions authority for its oversubscription criteria. +

+ )} {hasSixthForm && (
diff --git a/nextjs-app/components/school/SecondarySchoolSections.tsx b/nextjs-app/components/school/SecondarySchoolSections.tsx index 4dd0925..9c95439 100644 --- a/nextjs-app/components/school/SecondarySchoolSections.tsx +++ b/nextjs-app/components/school/SecondarySchoolSections.tsx @@ -13,7 +13,7 @@ import type { School, SchoolResult, AbsenceData, OfstedInspection, SchoolCensus, - SchoolAdmissions, SchoolDeprivation, SchoolFinance, NationalAverages, + SchoolAdmissions, SchoolAdmissionDistance, SchoolDeprivation, SchoolFinance, NationalAverages, } from '@/lib/types'; import { ofstedLegacyAreas } from '@/lib/utils'; import type { SecondaryFlags } from '@/lib/schoolSections'; @@ -32,6 +32,7 @@ export interface SecondarySchoolSectionsProps { ofsted: OfstedInspection | null; census: SchoolCensus | null; admissions: SchoolAdmissions | null; + admissionDistance: SchoolAdmissionDistance | null; deprivation: SchoolDeprivation | null; finance: SchoolFinance | null; nationalAvg: NationalAverages | null; @@ -40,7 +41,7 @@ export interface SecondarySchoolSectionsProps { export function SecondarySchoolSections({ schoolInfo, yearlyData, ofsted, census, - admissions, deprivation, finance, nationalAvg, flags, + admissions, admissionDistance, deprivation, finance, nationalAvg, flags, }: SecondarySchoolSectionsProps) { const secondaryAvg = nationalAvg?.secondary ?? {}; @@ -79,9 +80,12 @@ export function SecondarySchoolSections({ /> )} - {admissions && ( + {/* See PrimarySchoolSections: distance and EES admissions are independent + sources, so either one warrants the section. */} + {(admissions || admissionDistance) && ( diff --git a/nextjs-app/components/school/lastDistanceOffered.ts b/nextjs-app/components/school/lastDistanceOffered.ts new file mode 100644 index 0000000..5523384 --- /dev/null +++ b/nextjs-app/components/school/lastDistanceOffered.ts @@ -0,0 +1,61 @@ +/** + * Shared logic and copy for the "last distance offered" figure. + * + * The primary and secondary admissions sections lay their metrics out + * differently (a seamless tile grid vs. metric cards), so the markup is not + * shared — but the words are. Every caveat below is doing a specific job, and + * a figure that appeared on one template without them would be misleading in a + * way the other template was not: + * + * * The year, because a cut-off is the outcome of one particular admissions + * round and means nothing detached from it. + * * "not a fixed catchment", because that is the inference a parent will + * otherwise draw, and it is wrong — the distance moves every year. + * * The route count, because on a banded school the headline is the widest + * of several cut-offs and does not apply to every child. + */ + +import type { SchoolAdmissionDistance } from '@/lib/types'; +import { formatCutoffDistance, formatEntryYear } from '@/lib/utils'; + +export interface CutoffDisplay { + /** Headline figure, e.g. "0.31 miles". */ + primary: string; + /** Supporting conversion, e.g. "500 m". */ + secondary: string; + /** Entry point the figure belongs to, e.g. "September 2025". */ + entryYear: string; + /** Present only where several admission routes were collapsed into one figure. */ + routeNote: string | null; +} + +/** What the cut-off measures, and what it does not. Identical on both templates. */ +export const CUTOFF_NOTE = + 'The furthest home offered a place, after higher priorities such as siblings, ' + + 'faith and EHCP were applied. It is not a fixed catchment — it moves every year ' + + 'with the number of applications.'; + +/** Straight-line, because that is how councils measure it. */ +export const CUTOFF_MEASUREMENT_NOTE = 'Measured in a straight line from the school.'; + +export function describeCutoff( + distance: SchoolAdmissionDistance | null | undefined +): CutoffDisplay | null { + if (!distance) return null; + + const figure = formatCutoffDistance(distance.distance_m); + if (!figure) return null; + + const routes = distance.route_count ?? 1; + + return { + primary: figure.primary, + secondary: figure.secondary, + entryYear: formatEntryYear(distance.year), + routeNote: + routes > 1 + ? `Furthest of ${routes} admission routes at this school — the one that ` + + 'applies to your child may have had a shorter cut-off.' + : null, + }; +} diff --git a/nextjs-app/components/school/schoolSections.module.css b/nextjs-app/components/school/schoolSections.module.css index 6ea4edc..a9f4515 100644 --- a/nextjs-app/components/school/schoolSections.module.css +++ b/nextjs-app/components/school/schoolSections.module.css @@ -778,6 +778,38 @@ word-break: break-word; } +/* Last distance offered, secondary template. These cards are opaque + (--bg-secondary), so the tint can be set directly here — unlike the primary + tile grid, nothing shows through between them. */ +.metricCardDistance { + background-color: var(--bg-card); + background-image: linear-gradient(var(--brand-bg), var(--brand-bg)); + border-color: var(--brand); + /* Two columns' worth of width. Its label carries the entry year as well as + the measure, which at the 140px track width of the other cards wraps to + three lines and pushes the figure out of alignment with them. */ + grid-column: span 2; +} + +.metricCardDistance .metricValue { + flex-direction: column; + gap: 0.1rem; + color: var(--brand-strong); +} + +.metricValueSub { + font-size: 0.75rem; + font-weight: 500; + color: var(--text-muted); +} + +/* Same measured reason as .admissionsTileDistance above: --text-muted falls to + 4.09:1 on the tinted card in dark theme. */ +.metricCardDistance .metricLabel, +.metricCardDistance .metricValueSub { + color: var(--text-secondary); +} + .metricHint { font-size: 0.7rem; color: var(--text-muted); @@ -1453,6 +1485,62 @@ color: var(--brand-strong); } +/* ── Admissions: last distance offered ── + * + * Spans the full grid width instead of taking a fifth half-cell. Two reasons: + * the tiles are a 2-column grid, so a fifth tile would sit alone next to a gap; + * and this is the figure the section is most often opened for, which a + * half-width cell would rank below "Applied in total". + */ +.admissionsTileDistance { + grid-column: 1 / -1; + /* + * --brand-bg is translucent, and the tile grid paints --border between + * cells, so a plain `background` would let the hairline colour show through + * and read as muddy rather than tinted. The opaque card colour is laid down + * first and the tint composited over it as an image. + */ + background-color: var(--bg-card); + background-image: linear-gradient(var(--brand-bg), var(--brand-bg)); +} + +.admissionsTileDistance .admissionsTileNum { + color: var(--brand-strong); +} + +/* + * The tint costs the muted tokens their AA margin, so this tile does not use + * them. + * + * Measured on the composited backdrop, not the computed one: the tint is a + * background-image over --bg-card, so getComputedStyle reports the untinted + * card and flatters the result. Sampling the rendered pixel in dark theme gives + * a backdrop of rgb(34,59,64), on which --text-muted is 4.09:1 — under the 4.5 + * floor — while --text-secondary is 6.50:1. + * + * The label is where the year lives, so it is the last thing on the tile that + * can afford to be hard to read: a distance without its year is not a fact + * about anything. + */ +.admissionsTileDistance .admissionsTileLabel, +.admissionsTileDistance .admissionsTileSub { + color: var(--text-secondary); +} + +/* + * The caveats, not a footnote in the visual-noise sense: without them the + * figure reads as a catchment boundary. Kept at body contrast rather than + * --text-muted for that reason — it qualifies the number above it, so it has + * to be as readable as the number. + */ +.admissionsDistanceNote { + margin: 0.85rem 0 0; + font-size: 0.8125rem; + line-height: 1.55; + color: var(--text-secondary); + max-width: 68ch; +} + .admissionsVerdict { margin-top: 0.75rem; margin-bottom: 0.25rem; @@ -1568,6 +1656,20 @@ grid-template-rows: 1fr 1fr; } +/* The distance tile occupies a third row. Left implicit it would inherit the + 1fr sizing of the row template and stretch to match the metric tiles above, + which are twice its content height. */ +.admissionsViewYear .admissionsTilesWithDistance { + grid-template-rows: 1fr 1fr auto; +} + +/* Cut-off distance but no EES admissions figures: there is nothing to put in + the two metric rows, and leaving them in the template paints an empty block + of the grid's gap colour beneath the tile. */ +.admissionsViewYear .admissionsTilesDistanceOnly { + grid-template-rows: auto; +} + .admissionsChartCap { font-size: 0.8125rem; font-weight: 600; diff --git a/nextjs-app/lib/schoolSections.ts b/nextjs-app/lib/schoolSections.ts index e11b614..4a4dbc5 100644 --- a/nextjs-app/lib/schoolSections.ts +++ b/nextjs-app/lib/schoolSections.ts @@ -9,7 +9,7 @@ import type { School, SchoolResult, AbsenceData, SchoolCensus, - OfstedInspection, SchoolAdmissions, SchoolDeprivation, SchoolFinance, + OfstedInspection, SchoolAdmissions, SchoolAdmissionDistance, SchoolDeprivation, SchoolFinance, } from './types'; import { isSpecialSchool } from './utils'; @@ -119,6 +119,9 @@ export function computeSchoolFlags({ export interface NavItemsInput { ofsted: OfstedInspection | null; admissions: SchoolAdmissions | null; + /** A published cut-off distance renders the Admissions section on its own, + * so it has to open the nav entry too. */ + admissionDistance: SchoolAdmissionDistance | null; yearlyDataLength: number; } @@ -133,7 +136,7 @@ export interface NavItemsInput { */ export function buildNavItems( flags: SchoolFlags, - { ofsted, admissions, yearlyDataLength }: NavItemsInput, + { ofsted, admissions, admissionDistance, yearlyDataLength }: NavItemsInput, ): NavItem[] { const navItems: NavItem[] = []; if (ofsted) navItems.push({ id: 'ofsted', label: 'Ofsted' }); @@ -143,7 +146,7 @@ export function buildNavItems( label: flags.isAllThrough ? 'Results' : flags.isSecondary ? 'GCSEs' : 'SATs', }); } - if (admissions) navItems.push({ id: 'admissions', label: 'Admissions' }); + if (admissions || admissionDistance) navItems.push({ id: 'admissions', label: 'Admissions' }); if (flags.hasInclusionData) navItems.push({ id: 'inclusion', label: 'Pupils' }); if (yearlyDataLength > 0) navItems.push({ id: 'history', label: 'History' }); if (flags.hasSchoolLife) navItems.push({ id: 'school-life', label: 'School Life' }); @@ -213,12 +216,12 @@ export function computeSecondaryFlags({ */ export function buildSecondaryNavItems( flags: SecondaryFlags, - { ofsted, admissions, yearlyDataLength }: NavItemsInput, + { ofsted, admissions, admissionDistance, yearlyDataLength }: NavItemsInput, ): NavItem[] { const navItems: NavItem[] = []; if (ofsted) navItems.push({ id: 'ofsted', label: 'Ofsted' }); if (flags.hasResults) navItems.push({ id: 'gcse', label: 'GCSEs' }); - if (admissions) navItems.push({ id: 'admissions', label: 'Admissions' }); + if (admissions || admissionDistance) navItems.push({ id: 'admissions', label: 'Admissions' }); if (yearlyDataLength > 1) navItems.push({ id: 'history', label: 'History' }); if (flags.hasWellbeing) navItems.push({ id: 'wellbeing', label: 'Wellbeing' }); if (flags.hasFinance) navItems.push({ id: 'finances', label: 'Finances' }); diff --git a/nextjs-app/lib/types.ts b/nextjs-app/lib/types.ts index 5778f06..32ff142 100644 --- a/nextjs-app/lib/types.ts +++ b/nextjs-app/lib/types.ts @@ -163,6 +163,30 @@ export interface SchoolAdmissions { } +/** + * Last distance offered — how far from the school the last child to be given a + * place that year lived. + * + * Not a catchment: it is the outcome of one year's applications and moves every + * year. Only published by the local authorities that choose to, so most schools + * have none of this at all. + */ +export interface SchoolAdmissionDistance { + /** Entry year the cut-off belongs to — always shown alongside the figure. */ + year: number; + /** Straight-line distance in metres. */ + distance_m: number | null; + /** + * Admission routes collapsed into this figure (ability bands, or separate + * reception and junior intakes). Above 1 the distance is the furthest of + * several, which the page has to say out loud. + */ + route_count: number | null; + la_name: string | null; + /** The unit the council published in ("miles", "meters", "km"). */ + distance_unit_raw: string | null; +} + export interface SchoolDeprivation { lsoa_code: string | null; idaci_score: number | null; @@ -330,6 +354,8 @@ export interface SchoolDetailsResponse { admissions: SchoolAdmissions | null; /** All available admissions years, oldest first. Drives the multi-year trend view. */ admissions_history: SchoolAdmissions[]; + /** Latest published cut-off distance, or null where the LA publishes none. */ + admission_distance: SchoolAdmissionDistance | null; deprivation: SchoolDeprivation | null; finance: SchoolFinance | null; } diff --git a/nextjs-app/lib/utils.ts b/nextjs-app/lib/utils.ts index fcbaf41..45f3e78 100644 --- a/nextjs-app/lib/utils.ts +++ b/nextjs-app/lib/utils.ts @@ -502,6 +502,54 @@ export function getPhaseStyle(phase?: string | null): { key: string; label: stri } } +const METRES_PER_MILE = 1609.344; + +/** + * Format an admission cut-off distance. + * + * Miles lead because that is the unit councils publish in — 90% of the source + * rows are miles — and because it is the unit a parent has already been quoted + * if they have read their council's booklet. The metric figure follows as + * support rather than replacing it. + * + * Below 100 m the miles figure stops carrying information ("0.03 miles"), so + * the two swap round and metres lead instead. + */ +export function formatCutoffDistance( + metres: number | null | undefined +): { primary: string; secondary: string } | null { + if (metres == null || !Number.isFinite(metres) || metres <= 0) return null; + + const miles = metres / METRES_PER_MILE; + + if (metres < 100) { + return { + primary: `${Math.round(metres)} m`, + secondary: `${miles.toFixed(2)} miles`, + }; + } + + return { + primary: `${miles.toFixed(2)} miles`, + secondary: metres < 1000 + ? `${Math.round(metres / 10) * 10} m` + : `${(metres / 1000).toFixed(1)} km`, + }; +} + +/** + * The entry point a cut-off belongs to, phrased the way councils phrase it. + * + * The distance year is a plain entry year (2025 = the September 2025 intake), + * not the six-digit academic year EES uses, so formatAcademicYear would render + * it as "2025/26" and invite the reader to think of it as a school year rather + * than an application round. + */ +export function formatEntryYear(year: number | null | undefined): string { + if (year == null) return ''; + return `September ${year}`; +} + export function formatAcademicYear(year: number | null | undefined): string { if (year == null) return ''; const s = year.toString(); diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 0278b5d..82ad099 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -224,3 +224,68 @@ with DAG( ) extract_idaci >> dbt_build_idaci >> invalidate_cache_idaci + + +# ── Annual DAG (Last distance offered) ──────────────────────────────── + +with DAG( + dag_id="school_data_annual_distance", + default_args=default_args, + description="Last distance offered (LA admission cut-offs) extraction and transform", + # Councils publish on allocation day (March for secondary, April for + # primary) and each one on its own timetable, so there is no date worth + # scheduling against. Triggered manually after a collection run refreshes + # the CSV. + schedule=None, + start_date=datetime(2025, 1, 1), + catchup=False, + tags=["school-compare", "annual"], +) as annual_distance_dag: + + extract_distance = BashOperator( + task_id="extract_distance", + bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-school-distance target-postgres", + ) + + # Coverage is the thing that silently rots here: the CSV is assembled by + # hand from council publications, so a collection run that half-failed + # produces a valid file with a fraction of the schools in it. A row count + # alone would not catch that — losing an entire local authority leaves the + # total looking healthy — so the floor is checked on distinct LAs too. + validate_distance = BashOperator( + task_id="validate_distance", + bash_command=f""" + cd {PIPELINE_DIR} && python -c " +import psycopg2, os, sys +conn = psycopg2.connect( + host=os.environ.get('PG_HOST', 'localhost'), + port=os.environ.get('PG_PORT', '5432'), + user=os.environ.get('PG_USER', 'postgres'), + password=os.environ.get('PG_PASSWORD', 'postgres'), + dbname=os.environ.get('PG_DATABASE', 'school_compare'), +) +cur = conn.cursor() +cur.execute('SELECT count(*), count(distinct urn), count(distinct la_code) FROM raw.school_distance_offered') +rows, urns, las = cur.fetchone() +conn.close() +print(f'Loaded {{rows}} rows, {{urns}} schools, {{las}} local authorities') +if rows < 7000 or urns < 3000 or las < 45: + print('ERROR: distance extract is short of expected coverage ' + '(baseline 2026-08: 9128 rows / 3726 schools / 57 LAs)', file=sys.stderr) + sys.exit(1) +print('Validation passed') +" + """, + ) + + dbt_build_distance = BashOperator( + task_id="dbt_build", + bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_school_distance+ fact_admission_distance+", + ) + + invalidate_cache_distance = BashOperator( + task_id="invalidate_cache", + bash_command=INVALIDATE_CACHE_CMD, + ) + + extract_distance >> validate_distance >> dbt_build_distance >> invalidate_cache_distance diff --git a/pipeline/meltano.yml b/pipeline/meltano.yml index 856ac86..65c8199 100644 --- a/pipeline/meltano.yml +++ b/pipeline/meltano.yml @@ -67,6 +67,21 @@ plugins: pip_url: ./plugins/extractors/tap-uk-idaci executable: tap-uk-idaci + # Last distance offered — admission cut-offs collected from LA + # publications. There is no national source for this, so the URL points at + # the collected CSV rather than at a government endpoint (same pattern as + # the legacy KS2/KS4 URLs above). + - name: tap-uk-school-distance + namespace: uk_school_distance + pip_url: ./plugins/extractors/tap-uk-school-distance + executable: tap-uk-school-distance + settings: + - name: download_url + kind: string + description: URL of the collected last-distance-offered CSV + config: + download_url: http://10.0.1.224:8081/filebrowser/api/public/dl/UU_oEqVr?inline=true + loaders: - name: target-postgres variant: meltanolabs diff --git a/pipeline/plugins/extractors/tap-uk-school-distance/pyproject.toml b/pipeline/plugins/extractors/tap-uk-school-distance/pyproject.toml new file mode 100644 index 0000000..b363be1 --- /dev/null +++ b/pipeline/plugins/extractors/tap-uk-school-distance/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tap-uk-school-distance" +version = "0.1.0" +description = "Singer tap for last-distance-offered admission cut-offs collected from LA admissions publications" +requires-python = ">=3.10" +dependencies = [ + "singer-sdk~=0.53", + "requests>=2.31", +] + +[project.scripts] +tap-uk-school-distance = "tap_uk_school_distance.tap:TapUKSchoolDistance.cli" diff --git a/pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/__init__.py b/pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/__init__.py new file mode 100644 index 0000000..c4c3617 --- /dev/null +++ b/pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/__init__.py @@ -0,0 +1 @@ +"""tap-uk-school-distance: Singer tap for LA "last distance offered" admission cut-offs.""" diff --git a/pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/tap.py b/pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/tap.py new file mode 100644 index 0000000..2445141 --- /dev/null +++ b/pipeline/plugins/extractors/tap-uk-school-distance/tap_uk_school_distance/tap.py @@ -0,0 +1,135 @@ +"""Singer tap for "last distance offered" school admission cut-offs. + +Unlike every other tap in this project, the upstream here is not a single +government endpoint. There is no national dataset of admission cut-off +distances: each local authority publishes its own, in its own format +(PDF booklets, XLSX allocation tables, HTML pages), and only some publish +at all. The CSV this tap reads is the output of that collection work — +one row per school per year per admission route, already normalised to +metres in `distance_m`. + +The tap deliberately does no cleaning. Implausible values (0.0 miles, +1-metre cut-offs, a 533-mile outlier) are loaded verbatim into `raw` and +filtered in `stg_school_distance`, so the raw table stays a faithful +record of what the councils published and the plausibility rules live in +one reviewable place. +""" + +from __future__ import annotations + +import csv +import io + +import requests +from singer_sdk import Stream, Tap +from singer_sdk import typing as th + +REQUEST_TIMEOUT = 300 + + +class SchoolDistanceStream(Stream): + """Stream: last distance offered, one row per school × year × admission route.""" + + name = "school_distance_offered" + # school_name is part of the key, not decoration: where a school admits + # through several routes the route is encoded in the name ("… Band 1", + # "… Band 2"), and (urn, year) alone is not unique. Verified against the + # source: (urn, year) collides on 118 keys, (urn, year, school_name) on + # none. Without school_name in the key a reload would silently drop the + # other bands. + primary_keys = ["urn", "year", "school_name"] + replication_key = None + + schema = th.PropertiesList( + th.Property("urn", th.IntegerType, required=True), + th.Property("year", th.IntegerType, required=True), + th.Property("school_name", th.StringType, required=True), + th.Property("la_code", th.IntegerType), + th.Property("la_name", th.StringType), + # The figure as the council published it, before unit conversion — + # kept so a surprising metre value can be traced back to "0.31 miles" + # without reopening the source PDF. + th.Property("distance_value_raw", th.NumberType), + th.Property("distance_unit_raw", th.StringType), + th.Property("distance_m", th.NumberType), + # Path of the council publication the row was read from. This is the + # provenance trail for a figure we cannot re-derive from an API. + th.Property("source_file", th.StringType), + ).to_dict() + + @staticmethod + def _num(value: str) -> float | None: + try: + return float(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _int(value: str) -> int | None: + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + def get_records(self, context): + url = self.config["download_url"] + self.logger.info("Downloading school distance data from %s", url) + + resp = requests.get(url, timeout=REQUEST_TIMEOUT) + resp.raise_for_status() + + reader = csv.DictReader(io.StringIO(resp.text)) + + yielded = 0 + skipped = 0 + for row in reader: + urn = self._int(row.get("urn")) + year = self._int(row.get("year")) + name = (row.get("school_name") or "").strip() + + # Every part of the primary key must be present, or the load is + # not idempotent. One row in the source has a blank year. + if urn is None or year is None or not name: + skipped += 1 + continue + + yield { + "urn": urn, + "year": year, + "school_name": name, + "la_code": self._int(row.get("la_code")), + "la_name": (row.get("la_name") or "").strip() or None, + "distance_value_raw": self._num(row.get("distance_value_raw")), + "distance_unit_raw": (row.get("distance_unit_raw") or "").strip() or None, + "distance_m": self._num(row.get("distance_m")), + "source_file": (row.get("source_file") or "").strip() or None, + } + yielded += 1 + + self.logger.info( + "Yielded %d distance records (%d skipped: missing urn, year or school name)", + yielded, + skipped, + ) + + +class TapUKSchoolDistance(Tap): + """Singer tap for LA-published admission cut-off distances.""" + + name = "tap-uk-school-distance" + + config_jsonschema = th.PropertiesList( + th.Property( + "download_url", + th.StringType, + required=True, + description="URL of the collected last-distance-offered CSV", + ), + ).to_dict() + + def discover_streams(self): + return [SchoolDistanceStream(self)] + + +if __name__ == "__main__": + TapUKSchoolDistance.cli() diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 9869885..1aa86c1 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -156,6 +156,22 @@ models: - name: cross_la_offers - name: total_offers + - name: fact_admission_distance + description: > + Last distance offered — one row per URN per year. Where a school admits + through several routes (ability bands, separate reception/junior + intakes) the figure is the furthest of them and route_count says how + many were collapsed. + columns: + - name: urn + tests: [not_null] + - name: year + tests: [not_null] + - name: distance_m + tests: [not_null] + - name: route_count + tests: [not_null] + - name: fact_finance description: School financial data — one row per URN per year columns: diff --git a/pipeline/transform/models/marts/fact_admission_distance.sql b/pipeline/transform/models/marts/fact_admission_distance.sql new file mode 100644 index 0000000..dc413c4 --- /dev/null +++ b/pipeline/transform/models/marts/fact_admission_distance.sql @@ -0,0 +1,50 @@ +-- Mart: last distance offered — one row per URN per year. +-- +-- Collapses the per-route grain of stg_school_distance to the grain the site +-- serves. Where a school admits through several routes the figure shown is the +-- FURTHEST of them, which is what "last distance offered" literally means: the +-- distance of the last place the school gave out that year. +-- +-- route_count travels with it so the page can say so. A parent looking at a +-- banded secondary needs to know that 2.4 miles is the widest of four bands +-- and not the number that applies to their own child — the count is what makes +-- that sentence possible, and dropping it would make the figure quietly +-- optimistic. +-- +-- Kept separate from fact_admissions rather than joined into it. That mart is +-- EES-derived and near-complete for England; this one comes from 57 local +-- authorities that publish cut-offs at all, so folding it in would add mostly +-- null columns to every school in the country and tie two very different +-- refresh cadences together. + +with routes as ( + select * from {{ ref('stg_school_distance') }} +), + +per_year as ( + select + urn, + year, + max(distance_m) as distance_m, + count(*) as route_count, + min(la_code) as la_code, + min(la_name) as la_name, + -- The unit the council actually published in. Where routes disagree + -- (they do not in the current data) the alphabetically first is taken + -- rather than an arbitrary row, so the mart is deterministic. + min(distance_unit_raw) as distance_unit_raw, + min(source_file) as source_file + from routes + group by urn, year +) + +select + urn, + year, + distance_m, + route_count, + la_code, + la_name, + distance_unit_raw, + source_file +from per_year diff --git a/pipeline/transform/models/staging/_stg_sources.yml b/pipeline/transform/models/staging/_stg_sources.yml index b390790..6b294c4 100644 --- a/pipeline/transform/models/staging/_stg_sources.yml +++ b/pipeline/transform/models/staging/_stg_sources.yml @@ -61,3 +61,11 @@ sources: - name: idaci description: Income Deprivation Affecting Children Index lookups + + - name: school_distance_offered + description: > + Last distance offered (admission cut-offs) transcribed from local + authority admissions publications. One row per school × year × + admission route; no national source exists, so coverage is limited to + the LAs that publish. Loaded verbatim — implausible values are + filtered in stg_school_distance, not here. diff --git a/pipeline/transform/models/staging/stg_school_distance.sql b/pipeline/transform/models/staging/stg_school_distance.sql new file mode 100644 index 0000000..705ecdd --- /dev/null +++ b/pipeline/transform/models/staging/stg_school_distance.sql @@ -0,0 +1,55 @@ +-- Staging model: "last distance offered" admission cut-offs. +-- +-- One row per school × year × admission route. A school admits through +-- several routes when it bands by ability (Islington publishes four bands per +-- secondary) or takes more than one intake (Gloucestershire publishes +-- reception, junior and secondary allocations separately). The route is +-- carried in school_name, which is why it forms part of the grain here and is +-- collapsed only in fact_admission_distance. +-- +-- PLAUSIBILITY BAND (25 m – 25 km) +-- +-- The source is transcribed from council PDFs, spreadsheets and web pages, and +-- a minority of rows are transcription or unit errors rather than real +-- cut-offs. Three failure modes, all present: +-- +-- * 0.0 miles (34 rows, mostly Staffordshire). Published where a school +-- filled on a higher criterion so distance was never reached — a real +-- fact, but "0 metres" read as a cut-off would tell a parent the opposite +-- of the truth. +-- * 1–10 metre values (72 rows, e.g. Hillingdon). A cut-off inside the +-- school gate is not physically possible; these are unit slips. +-- * Values above 25 km (~240 rows), topping out at 533 miles. Genuine rural +-- secondary cut-offs reach roughly 12–15 miles, so the ceiling keeps those +-- while cutting figures that are visibly wrong. +-- +-- Together these are ~4% of rows. They are loaded verbatim into raw and +-- excluded here, so the rule is reviewable in one place and the raw table +-- stays a faithful record of what each council published. + +{% set min_plausible_m = 25 %} +{% set max_plausible_m = 25000 %} + +with source as ( + select * from {{ source('raw', 'school_distance_offered') }} +), + +renamed as ( + select + cast(urn as integer) as urn, + cast(year as integer) as year, + trim(school_name) as admission_route, + cast(la_code as integer) as la_code, + la_name, + cast(distance_value_raw as numeric) as distance_value_raw, + distance_unit_raw, + cast(distance_m as numeric) as distance_m, + source_file + from source + where urn is not null + and year is not null + and distance_m is not null + and cast(distance_m as numeric) between {{ min_plausible_m }} and {{ max_plausible_m }} +) + +select * from renamed diff --git a/pipeline/transform/tests/assert_admission_distance_plausible.sql b/pipeline/transform/tests/assert_admission_distance_plausible.sql new file mode 100644 index 0000000..45f377c --- /dev/null +++ b/pipeline/transform/tests/assert_admission_distance_plausible.sql @@ -0,0 +1,27 @@ +-- Custom test: every published cut-off distance is inside the plausibility +-- band, and there is exactly one row per school per year. +-- +-- The band is enforced in stg_school_distance; this asserts it survived the +-- max() collapse in the mart. A failure means either the staging filter was +-- edited without updating this test, or the grain broke — both of which put a +-- visibly wrong number ("533 miles") on a live school page, which is the one +-- outcome this feature cannot afford. + +with out_of_band as ( + select urn, year, distance_m, 'out of plausibility band' as problem + from {{ ref('fact_admission_distance') }} + where distance_m is null + or distance_m < 25 + or distance_m > 25000 +), + +duplicated as ( + select urn, year, max(distance_m) as distance_m, 'duplicate urn/year' as problem + from {{ ref('fact_admission_distance') }} + group by urn, year + having count(*) > 1 +) + +select * from out_of_band +union all +select * from duplicated -- 2.54.0