feat(seo): split the sitemap into a per-family index

Search Console reports coverage per submitted sitemap, so one file per page
family is what will make W2's location pages measurable when they land. The
index's lastmod is generation time, which is the correct semantic there —
unlike on a <url>, where it would be a claim we cannot support.

Children sit under /sitemaps/ because Next only treats a whole bracketed path
segment as dynamic; a route folder named sitemap-[...parts] would be read as a
literal static segment and never match. Confirmed by the build output, which
lists /sitemaps/[...parts] as a dynamic route.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015mWQnpye9F299NVRCCSRvj

backend/app.py

@@ -7,6 +7,7 @@ Uses real data from UK Government Compare School Performance downloads.
 import hashlib
 import re
 from contextlib import asynccontextmanager
+from datetime import datetime, timezone
 from typing import Optional
 
 import numpy as np
@@ -48,11 +49,14 @@ PHASE_GROUPS: dict[str, set[str]] = {
     "all-through": {"all-through"},
 }
 
-BASE_URL = "https://schoolcompare.co.uk"
+# Must match SITE_URL in nextjs-app/lib/site.ts. The apex 301s to www, and a
+# sitemap <loc> that redirects wastes a crawl on every URL it lists.
+BASE_URL = "https://www.schoolcompare.co.uk"
 MAX_SLUG_LENGTH = 60
 
-# In-memory sitemap cache
+# In-memory sitemap cache: name -> XML. Populated on startup and by the admin
-_sitemap_xml: str | None = None
+# regenerate endpoint after a pipeline run.
+_sitemaps: dict[str, str] | None = None
 
 
 def _slugify(text: str) -> str:
@@ -70,43 +74,128 @@ def _school_url(urn: int, school_name: str) -> str:
     return f"/school/{urn}-{slug}"
 
 
-def build_sitemap() -> str:
+# Routes worth submitting that are not a school page. /admissions was missing
-    """Generate sitemap XML from in-memory school data. Returns the XML string."""
+# from the sitemap entirely despite being a static, indexable guide.
+STATIC_SITEMAP_PATHS = ("/", "/rankings", "/compare", "/admissions")
+
+
+def _has_publishable_data(row) -> bool:
+    """True when a school page has something a search result could state.
+
+    A school with no results in any year and no Ofsted grade renders an empty
+    page. Submitting it spends crawl budget and drags the corpus-wide quality
+    signal down, so it stays out of the sitemap. The page itself still resolves
+    for anyone who has the URL.
+    """
+    for field in ("rwm_expected_pct", "attainment_8_score", "ofsted_grade"):
+        value = row.get(field)
+        if value is not None and not pd.isna(value):
+            return True
+    return False
+
+
+def _url_element(loc: str, lastmod: str | None = None) -> str:
+    """One <url> entry. No priority or changefreq — Google ignores both."""
+    body = f"<loc>{loc}</loc>"
+    if lastmod:
+        body += f"<lastmod>{lastmod}</lastmod>"
+    return f"  <url>{body}</url>"
+
+
+def _school_sitemap_rows(df) -> list[str]:
+    """A <url> element per school that has something to show.
+
+    lastmod comes from the school's Ofsted date where there is one and is
+    omitted otherwise. An always-now lastmod is a claim Google learns to
+    distrust; an absent one honestly means "unknown".
+    """
+    if df.empty or "urn" not in df.columns or "school_name" not in df.columns:
+        return []
+
+    rows: list[str] = []
+    seen: set[int] = set()
+
+    # Latest row per URN first, so a school's most recent Ofsted date wins.
+    ordered = df.sort_values("year", ascending=False) if "year" in df.columns else df
+
+    for _, row in ordered.iterrows():
+        urn = int(row["urn"])
+        if urn in seen:
+            continue
+        seen.add(urn)
+        if not _has_publishable_data(row):
+            continue
+
+        lastmod = None
+        ofsted_date = row.get("ofsted_date")
+        if ofsted_date is not None and not pd.isna(ofsted_date):
+            lastmod = pd.Timestamp(ofsted_date).date().isoformat()
+
+        rows.append(_url_element(
+            BASE_URL + _school_url(urn, str(row["school_name"])), lastmod))
+
+    return rows
+
+
+# Sitemaps cap at 50,000 URLs per file. 10,000 keeps a child small enough to
+# scan by eye in Search Console, which is the point of splitting at all:
+# coverage is reported per submitted sitemap, so one file per page family is
+# what makes an indexation problem attributable to a family.
+SITEMAP_CHUNK_SIZE = 10_000
+
+# Children are served under /sitemaps/ because Next.js only treats a whole
+# bracketed path segment as dynamic — a route folder named "sitemap-[...parts]"
+# is read as a literal static segment and never matches.
+SITEMAP_CHILD_PREFIX = "/sitemaps"
+
+
+def _urlset(rows: list[str]) -> str:
+    return "\n".join([
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+        *rows,
+        "</urlset>",
+    ])
+
+
+def build_sitemaps() -> dict[str, str]:
+    """Build the sitemap index and every child, keyed by name."""
     df = load_school_data()
 
-    static_urls = [
+    children: dict[str, str] = {
-        (BASE_URL + "/",          "daily",   "1.0"),
+        "static.xml": _urlset(
-        (BASE_URL + "/rankings",  "weekly",  "0.8"),
+            [_url_element(BASE_URL + path) for path in STATIC_SITEMAP_PATHS]),
-        (BASE_URL + "/compare",   "weekly",  "0.8"),
+    }
+
+    school_rows = _school_sitemap_rows(df)
+    # Always emit at least one school child, so the index shape is stable even
+    # on an empty database.
+    chunks = [school_rows[i:i + SITEMAP_CHUNK_SIZE]
+              for i in range(0, len(school_rows), SITEMAP_CHUNK_SIZE)] or [[]]
+    for n, chunk in enumerate(chunks, start=1):
+        children[f"schools-{n}.xml"] = _urlset(chunk)
+
+    # On a sitemap index, lastmod means "when this sitemap file last changed",
+    # so generation time is the correct value here — unlike on a <url>, where
+    # it would be a claim about content we cannot support.
+    generated = datetime.now(timezone.utc).date().isoformat()
+    index_rows = [
+        f"  <sitemap><loc>{BASE_URL}{SITEMAP_CHILD_PREFIX}/{name}</loc>"
+        f"<lastmod>{generated}</lastmod></sitemap>"
+        for name in children
     ]
+    index = "\n".join([
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+        *index_rows,
+        "</sitemapindex>",
+    ])
+    return {**children, "sitemap.xml": index}
 
-    lines = ['<?xml version="1.0" encoding="UTF-8"?>',
-             '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
 
-    for url, freq, priority in static_urls:
+def build_sitemap() -> str:
-        lines.append(
+    """The sitemap index. Kept for `lifespan` and the admin endpoint."""
-            f"  <url><loc>{url}</loc>"
+    return build_sitemaps()["sitemap.xml"]
-            f"<changefreq>{freq}</changefreq>"
-            f"<priority>{priority}</priority></url>"
-        )
-
-    if not df.empty and "urn" in df.columns and "school_name" in df.columns:
-        seen = set()
-        for _, row in df[["urn", "school_name"]].drop_duplicates(subset="urn").iterrows():
-            urn = int(row["urn"])
-            name = str(row["school_name"])
-            if urn in seen:
-                continue
-            seen.add(urn)
-            path = _school_url(urn, name)
-            lines.append(
-                f"  <url><loc>{BASE_URL}{path}</loc>"
-                f"<changefreq>monthly</changefreq>"
-                f"<priority>0.6</priority></url>"
-            )
-
-    lines.append("</urlset>")
-    return "\n".join(lines)
 
 
 def clean_filter_values(series: pd.Series) -> list[str]:
@@ -284,7 +373,7 @@ def validate_postcode(postcode: Optional[str]) -> Optional[str]:
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Application lifespan - startup and shutdown events."""
-    global _sitemap_xml
+    global _sitemaps
     print("Loading school data from marts...")
     df = load_school_data()
     if df.empty:
@@ -294,9 +383,9 @@ async def lifespan(app: FastAPI):
         # Pre-compute the latest-year snapshot so the first search request is fast
         await asyncio.to_thread(load_latest_school_data)
     try:
-        _sitemap_xml = build_sitemap()
+        _sitemaps = build_sitemaps()
-        n = _sitemap_xml.count("<url>")
+        n = sum(x.count("<url>") for x in _sitemaps.values())
-        print(f"Sitemap built: {n} URLs.")
+        print(f"Sitemaps built: {len(_sitemaps)} files, {n} URLs.")
     except Exception as e:
         print(f"Warning: sitemap build failed on startup: {e}")
 
@@ -1087,16 +1176,28 @@ async def robots_txt():
     return FileResponse(settings.frontend_dir / "robots.txt", media_type="text/plain")
 
 
-@app.get("/sitemap.xml")
+def _serve_sitemap(name: str) -> Response:
-async def sitemap_xml():
+    global _sitemaps
-    """Serve sitemap.xml for search engine indexing."""
+    if _sitemaps is None:
-    global _sitemap_xml
-    if _sitemap_xml is None:
         try:
-            _sitemap_xml = build_sitemap()
+            _sitemaps = build_sitemaps()
         except Exception as e:
             raise HTTPException(status_code=503, detail=f"Sitemap unavailable: {e}")
-    return Response(content=_sitemap_xml, media_type="application/xml")
+    if name not in _sitemaps:
+        raise HTTPException(status_code=404, detail="No such sitemap")
+    return Response(content=_sitemaps[name], media_type="application/xml")
+
+
+@app.get("/sitemap.xml")
+async def sitemap_xml():
+    """Serve the sitemap index."""
+    return _serve_sitemap("sitemap.xml")
+
+
+@app.get("/sitemaps/{name}")
+async def sitemap_child(name: str):
+    """Serve a child sitemap (static.xml, or schools-N.xml)."""
+    return _serve_sitemap(name)
 
 
 @app.post("/api/admin/regenerate-sitemap")
@@ -1106,10 +1207,10 @@ async def regenerate_sitemap(
     _: bool = Depends(verify_admin_api_key),
 ):
     """Rebuild and cache the sitemap from current school data. Called by Airflow after data updates."""
-    global _sitemap_xml
+    global _sitemaps
-    _sitemap_xml = build_sitemap()
+    _sitemaps = build_sitemaps()
-    n = _sitemap_xml.count("<url>")
+    n = sum(x.count("<url>") for x in _sitemaps.values())
-    return {"status": "ok", "urls": n}
+    return {"status": "ok", "urls": n, "sitemaps": len(_sitemaps)}
 
 
 # Mount static files directly (must be after all routes to avoid catching API calls)

backend/tests/test_sitemap.py

@@ -0,0 +1,178 @@
+"""Tests for sitemap generation (spec 2026-08-20, workstream W1).
+
+The sitemap is built from the in-memory school DataFrame, so these inject a
+small frame via monkeypatch rather than touching a database.
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+
+def _schools_df() -> pd.DataFrame:
+    """Two schools: one with results, one with neither results nor Ofsted."""
+    base = {
+        "local_authority": "Testshire",
+        "school_type": "Academy",
+        "phase": "Primary",
+        "year": 202425,
+        "ofsted_date": None,
+    }
+    return pd.DataFrame(
+        [
+            {**base, "urn": 100001, "school_name": "Alpha Primary",
+             "rwm_expected_pct": 62.0, "attainment_8_score": np.nan,
+             "ofsted_grade": 2.0},
+            {**base, "urn": 100002, "school_name": "Ghost Primary",
+             "rwm_expected_pct": np.nan, "attainment_8_score": np.nan,
+             "ofsted_grade": np.nan},
+        ]
+    )
+
+
+@pytest.fixture()
+def sitemap(monkeypatch) -> str:
+    """The sitemap index."""
+    from backend import app as app_module
+
+    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
+    return app_module.build_sitemap()
+
+
+@pytest.fixture()
+def schools_child(monkeypatch) -> str:
+    """The first school child sitemap, where school URLs actually live."""
+    from backend import app as app_module
+
+    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
+    return app_module.build_sitemaps()["schools-1.xml"]
+
+
+@pytest.fixture()
+def static_child(monkeypatch) -> str:
+    from backend import app as app_module
+
+    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
+    return app_module.build_sitemaps()["static.xml"]
+
+
+def test_every_loc_uses_the_www_host(sitemaps):
+    # The apex 301s to www. A <loc> that redirects burns a crawl per URL.
+    # Checked across every file, index included, not just one.
+    for name, xml in sitemaps.items():
+        assert "https://www.schoolcompare.co.uk" in xml, name
+        assert "https://schoolcompare.co.uk" not in xml, name
+
+
+def test_school_with_results_is_listed(schools_child):
+    assert "/school/100001-alpha-primary" in schools_child
+
+
+def test_school_with_no_results_and_no_ofsted_is_omitted(schools_child):
+    # Nothing for a search result to say about it. Submitting it spends crawl
+    # budget and drags the corpus-wide quality signal down.
+    #
+    # Asserted against the child, not the index: the index carries no school
+    # URLs at all, so it would pass this trivially and prove nothing.
+    assert "/school/100002" not in schools_child
+
+
+def test_no_invented_priority_or_changefreq(sitemaps):
+    # Google ignores both. They were noise dressed as signal.
+    for name, xml in sitemaps.items():
+        assert "<priority>" not in xml, name
+        assert "<changefreq>" not in xml, name
+
+
+def test_ofsted_date_becomes_lastmod(monkeypatch):
+    from backend import app as app_module
+    import datetime
+
+    def _df():
+        base = _schools_df()
+        base.loc[base["urn"] == 100001, "ofsted_date"] = datetime.date(2024, 3, 14)
+        return base
+
+    monkeypatch.setattr(app_module, "load_school_data", _df)
+    xml = app_module.build_sitemaps()["schools-1.xml"]
+    assert "<lastmod>2024-03-14</lastmod>" in xml
+
+
+def test_no_lastmod_invented_when_date_unknown(monkeypatch):
+    # An always-now lastmod is a claim Google learns to distrust. Absent
+    # honestly means unknown.
+    from backend import app as app_module
+
+    def _df():
+        df = _schools_df()
+        df["ofsted_date"] = None
+        return df
+
+    monkeypatch.setattr(app_module, "load_school_data", _df)
+    # The child only. The index legitimately carries a lastmod, because there
+    # it means "when this sitemap file changed", which we do know.
+    xml = app_module.build_sitemaps()["schools-1.xml"]
+    assert "<lastmod>" not in xml
+
+
+def test_static_routes_are_listed(static_child):
+    for path in ("/", "/rankings", "/compare", "/admissions"):
+        assert f"<loc>https://www.schoolcompare.co.uk{path}</loc>" in static_child
+
+
+@pytest.fixture()
+def sitemaps(monkeypatch) -> dict:
+    from backend import app as app_module
+
+    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
+    return app_module.build_sitemaps()
+
+
+def test_index_lists_each_child(sitemaps):
+    index = sitemaps["sitemap.xml"]
+    assert "<sitemapindex" in index
+    assert "https://www.schoolcompare.co.uk/sitemaps/static.xml" in index
+    assert "https://www.schoolcompare.co.uk/sitemaps/schools-1.xml" in index
+
+
+def test_index_carries_no_url_elements(sitemaps):
+    # A sitemap index holds <sitemap> entries only; mixing in <url> is invalid.
+    assert "<url>" not in sitemaps["sitemap.xml"]
+
+
+def test_index_does_not_list_itself(sitemaps):
+    assert "<loc>https://www.schoolcompare.co.uk/sitemap.xml</loc>" not in sitemaps["sitemap.xml"]
+
+
+def test_static_child_holds_the_static_routes(sitemaps):
+    static = sitemaps["static.xml"]
+    for path in ("/", "/rankings", "/compare", "/admissions"):
+        assert f"<loc>https://www.schoolcompare.co.uk{path}</loc>" in static
+
+
+def test_school_child_holds_the_schools(sitemaps):
+    assert "/school/100001-alpha-primary" in sitemaps["schools-1.xml"]
+
+
+def test_children_are_chunked_under_the_limit(monkeypatch):
+    # Sitemaps cap at 50,000 URLs per file. Chunk at 10,000 so a child stays
+    # small enough to eyeball in Search Console.
+    from backend import app as app_module
+    import pandas as _pd
+
+    rows = [
+        {"urn": 200000 + i, "school_name": f"School {i}", "year": 202425,
+         "rwm_expected_pct": 60.0, "attainment_8_score": None,
+         "ofsted_grade": 2.0, "ofsted_date": None}
+        for i in range(10_001)
+    ]
+    monkeypatch.setattr(app_module, "load_school_data", lambda: _pd.DataFrame(rows))
+
+    maps = app_module.build_sitemaps()
+    assert maps["schools-1.xml"].count("<url>") == 10_000
+    assert maps["schools-2.xml"].count("<url>") == 1
+
+
+def test_build_sitemap_still_returns_the_index(sitemap):
+    # lifespan and the admin endpoint call build_sitemap(); keep it working.
+    assert "<sitemapindex" in sitemap

e2e/tests/journeys.spec.ts

@@ -1252,7 +1252,11 @@ test('a published cut-off distance is shown with the year it belongs to', async
   await expect(label).toContainText(`September ${found!.distance.year}`);
 
   // And the figure itself, in the unit councils publish in.
-  await expect(page.getByText(/\d+(\.\d+)? (miles|m)\b/).first()).toBeVisible();
+  // No trailing \b: the tile's support figure is an adjacent text node, so
+  // the element reads "0.88 miles 1.4 km" and a word boundary after "miles"
+  // is not guaranteed. The leading shape is what matters — a decimal figure
+  // in miles, never a metric one.
+  await expect(page.getByText(/\d+\.\d+ miles/).first()).toBeVisible();
 });
 
 test('a cut-off distance is never shown without saying it is not a catchment', async ({ page }) => {
@@ -1340,7 +1344,27 @@ test('a school page shows no year-by-year cut-off record', async ({ page }) => {
   // history back without the API.
   await expect(page.getByText(/Last distance offered, by year/)).toHaveCount(0);
   await expect(page.getByText(/too few to read as a trend/)).toHaveCount(0);
-  await expect(page.getByRole('button', { name: 'Distance' })).toHaveCount(0);
+
+  /*
+   * No "Distance" tab in the admissions segmented control.
+   *
+   * Scoped to the control, and exact, because getByRole matches accessible
+   * names by case-insensitive SUBSTRING: an unscoped { name: 'Distance' }
+   * matched "Show this distance on a map" — the map toggle added by this same
+   * feature — and failed a page that was entirely correct. Naming the group
+   * this assertion is about also means unrelated copy elsewhere on the page
+   * can never break it again.
+   */
+  const viewToggle = page.getByRole('group', { name: 'Admissions view' });
+  const tabs = await viewToggle.getByRole('button').allTextContents();
+
+  // Read the tabs positively rather than asserting an absence against a
+  // locator that might resolve to nothing: if the group selector ever stops
+  // matching, an absence check passes for the wrong reason, which is how the
+  // bug this test is guarding would slip back in unnoticed.
+  expect(tabs, 'admissions view toggle did not resolve').toContain('This year');
+  expect(tabs.map((s) => s.trim()), `admissions tabs: ${tabs.join(', ')}`)
+    .not.toContain('Distance');
 });
 
 test('the postcode check answers for the published year, and names it', async ({ page }) => {
@@ -1354,7 +1378,7 @@ test('the postcode check answers for the published year, and names it', async ({
   await expect(section).toBeVisible({ timeout: 15_000 });
 
   await page.getByLabel('Your postcode').fill('SW1A 1AA');
-  await page.getByRole('button', { name: 'Check' }).click();
+  await page.getByRole('button', { name: 'Check', exact: true }).click();
 
   const result = page.getByRole('status');
   const error = page.getByRole('alert');
@@ -1521,6 +1545,32 @@ test('searching a Welsh authority by name returns no schools', async ({ page })
   expect(body.total ?? (body.schools ?? []).length).toBe(0);
 });
 
+test('a Welsh school GIAS does not type as Welsh is gone too', async ({ page }) => {
+  // Beechwood College, Sully, CF64 5SE. GIAS types it "Special post 16
+  // institution" (32), not "Welsh establishment" (30), so the establishment
+  // type filter alone left it behind — it was the last Welsh school standing.
+  const res = await page.request.get('/api/schools?local_authority=Vale%20of%20Glamorgan&page_size=1');
+  expect(res.ok()).toBeTruthy();
+  const body = await res.json();
+  expect(body.total ?? (body.schools ?? []).length).toBe(0);
+
+  const page404 = await page.goto('/school/142458-beechwood-college');
+  expect(page404?.status()).toBe(404);
+});
+
+test('English schools with Welsh postcodes are kept', async ({ page }) => {
+  // Royal Mail postcode areas straddle the border: these Gloucestershire
+  // schools carry NP16/NP25 postcodes but are English schools with English
+  // data. A postcode-based filter would have wrongly deleted them, so this
+  // guards the fix from being "simplified" into one.
+  for (const urn of [115639, 115655, 137382]) {
+    const res = await page.request.get(`/api/schools/${urn}`);
+    expect(res.ok(), `URN ${urn} should still resolve`).toBeTruthy();
+    const { school_info } = await res.json();
+    expect(school_info.local_authority).toBe('Gloucestershire');
+  }
+});
+
 test('a Welsh school URL 404s while an English one still resolves', async ({ page }) => {
   // Paired on purpose: the Welsh assertion alone would also pass if the whole
   // site were down, which is the failure this test most needs to distinguish.
@@ -1537,16 +1587,111 @@ test('a Welsh school URL 404s while an English one still resolves', async ({ pag
   expect(welsh?.status(), 'a Welsh school should no longer resolve').toBe(404);
 });
 
-test('the sitemap submits no Welsh or overseas school', async ({ page }) => {
+async function sitemapChildren(page: Page): Promise<string[]> {
   const res = await page.request.get('/sitemap.xml');
   expect(res.ok()).toBeTruthy();
-  const xml = await res.text();
+  const index = await res.text();
+  expect(index).toContain('<sitemapindex');
+  return [...index.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
+}
 
-  const urlCount = (xml.match(/<url>/g) ?? []).length;
+test('the sitemap index names children that all resolve', async ({ page }) => {
-  expect(urlCount, 'sitemap looks empty or truncated').toBeGreaterThan(1000);
+  const index = await (await page.request.get('/sitemap.xml')).text();
+  // An index holds <sitemap> entries only; mixing in <url> is invalid.
+  expect(index).not.toContain('<url>');
 
-  // 401559 (Cardiff) and 402426 (ACT Schools, Cardiff) were both submitted
+  const locs = await sitemapChildren(page);
-  // before the England-only filter landed.
+  expect(locs.length).toBeGreaterThanOrEqual(2);
-  expect(xml).not.toContain('/school/401559');
+
-  expect(xml).not.toContain('/school/402426');
+  for (const loc of locs) {
+    expect(loc.startsWith('https://www.schoolcompare.co.uk/sitemaps/')).toBeTruthy();
+    const child = await page.request.get(new URL(loc).pathname);
+    expect(child.ok(), `${loc} should resolve`).toBeTruthy();
+    expect(await child.text()).toContain('<urlset');
+  }
+});
+
+test('the sitemap submits no Welsh or overseas school', async ({ page }) => {
+  const locs = await sitemapChildren(page);
+
+  let total = 0;
+  for (const loc of locs) {
+    const xml = await (await page.request.get(new URL(loc).pathname)).text();
+    total += (xml.match(/<url>/g) ?? []).length;
+    // 401559 (Adamsdown, Cardiff) and 402426 (ACT Schools, Cardiff) were both
+    // submitted before the England-only filter landed.
+    expect(xml).not.toContain('/school/401559');
+    expect(xml).not.toContain('/school/402426');
+  }
+  expect(total, 'sitemap looks empty or truncated').toBeGreaterThan(1000);
+});
+
+test('the sitemap invents no priority or changefreq', async ({ page }) => {
+  const [first] = await sitemapChildren(page);
+  expect(first).toBeTruthy();
+
+  const xml = await (await page.request.get(new URL(first).pathname)).text();
+  // Google ignores both. They were noise dressed as signal.
+  expect(xml).not.toContain('<priority>');
+  expect(xml).not.toContain('<changefreq>');
+});
+
+/*
+ * Canonical URLs (spec 2026-08-20, W1).
+ *
+ * Every indexable route declares exactly one canonical, on the www host, with
+ * no query string. The homepage's eleven search params filter a result set
+ * rather than making a new document, so they all collapse onto "/".
+ */
+const CANONICAL_ROUTES: Array<[string, string]> = [
+  ['/', 'https://www.schoolcompare.co.uk/'],
+  ['/rankings', 'https://www.schoolcompare.co.uk/rankings'],
+  ['/admissions', 'https://www.schoolcompare.co.uk/admissions'],
+];
+
+for (const [path, expected] of CANONICAL_ROUTES) {
+  test(`${path} declares exactly one canonical, on the www host`, async ({ page }) => {
+    await page.goto(path);
+    const hrefs = await page.locator('link[rel="canonical"]').evaluateAll(
+      (els) => els.map((e) => e.getAttribute('href')));
+    expect(hrefs, `${path} should declare one canonical`).toHaveLength(1);
+    expect(hrefs[0]).toBe(expected);
+  });
+}
+
+test('a filtered homepage still canonicalises to the bare root', async ({ page }) => {
+  await page.goto('/?search=primary&phase=primary&sort=name&page=2');
+  const href = await page.locator('link[rel="canonical"]').first()
+    .getAttribute('href');
+  expect(href).toBe('https://www.schoolcompare.co.uk/');
+});
+
+test('a school page canonicalises to its own slug on the www host', async ({ page }) => {
+  const res = await page.request.get('/api/schools?search=primary&per_page=1');
+  expect(res.ok()).toBeTruthy();
+  const [first] = (await res.json()).schools ?? [];
+  expect(first, 'no school available').toBeTruthy();
+
+  await page.goto(`/school/${first.urn}-x`);
+  const href = await page.locator('link[rel="canonical"]').first()
+    .getAttribute('href');
+  expect(href).toMatch(/^https:\/\/www\.schoolcompare\.co\.uk\/school\/\d+-/);
+});
+
+test('a bare /compare is indexable, a parameterised one is not', async ({ page }) => {
+  await page.goto('/compare');
+  await expect(page.locator('meta[name="robots"]')).toHaveCount(0);
+
+  const [a, b] = await twoPrimaryUrns(page);
+  await page.goto(`/compare?urns=${a},${b}`);
+  const robots = await page.locator('meta[name="robots"]').first()
+    .getAttribute('content');
+  expect(robots).toContain('noindex');
+  expect(robots).toContain('follow');
+
+  // noindex but follow: the links out to each school page still count, so the
+  // canonical must still be present and point at the bare path.
+  const canonical = await page.locator('link[rel="canonical"]').first()
+    .getAttribute('href');
+  expect(canonical).toBe('https://www.schoolcompare.co.uk/compare');
 });

nextjs-app/__tests__/app/metadata.test.ts

@@ -0,0 +1,53 @@
+import { metadata as homeMetadata } from '@/app/page';
+import { metadata as rankingsMetadata } from '@/app/rankings/page';
+import { metadata as admissionsMetadata } from '@/app/admissions/page';
+import { generateMetadata as compareMetadata } from '@/app/compare/page';
+
+describe('canonical URLs', () => {
+  it('the homepage canonicalises to the bare root', () => {
+    // page.tsx reads eleven search params. Without this, every filter
+    // combination is a crawlable near-duplicate of the one page we want to
+    // rank for "compare schools".
+    expect(homeMetadata.alternates?.canonical)
+      .toBe('https://www.schoolcompare.co.uk/');
+  });
+
+  it('rankings canonicalises to the bare path', () => {
+    expect(rankingsMetadata.alternates?.canonical)
+      .toBe('https://www.schoolcompare.co.uk/rankings');
+  });
+
+  it('admissions canonicalises to the bare path', () => {
+    expect(admissionsMetadata.alternates?.canonical)
+      .toBe('https://www.schoolcompare.co.uk/admissions');
+  });
+});
+
+describe('/compare indexability', () => {
+  it('the bare compare page is indexable and canonical to itself', async () => {
+    // This is the landing page for the "compare schools" head term.
+    const meta = await compareMetadata({ searchParams: Promise.resolve({}) });
+    expect(meta.alternates?.canonical)
+      .toBe('https://www.schoolcompare.co.uk/compare');
+    expect(meta.robots).toBeUndefined();
+  });
+
+  it('a comparison of specific schools is noindex, follow', async () => {
+    // ~317 million pairs before triples. Indexing the parameter space would
+    // swamp everything else in the corpus.
+    const meta = await compareMetadata({
+      searchParams: Promise.resolve({ urns: '100001,100002' }),
+    });
+    expect(meta.robots).toEqual({ index: false, follow: true });
+  });
+
+  it('a parameterised comparison still canonicalises to the bare path', async () => {
+    // follow:true plus a canonical means the outbound links to each school
+    // page still pass value even though this URL is not indexed.
+    const meta = await compareMetadata({
+      searchParams: Promise.resolve({ urns: '100001,100002' }),
+    });
+    expect(meta.alternates?.canonical)
+      .toBe('https://www.schoolcompare.co.uk/compare');
+  });
+});

nextjs-app/__tests__/components/lastDistanceOffered.test.tsx

@@ -30,6 +30,17 @@ describe('primary detail page', () => {
     expect(screen.getByText(/Last distance offered/)).toHaveTextContent('September 2024');
   });
 
+  it('keeps the figure and its metric support readable as two numbers', () => {
+    // They are flex children with a CSS gap and nothing between them in the
+    // text layer, which read as "0.48 miles770 m" to a screen reader and to any
+    // text matcher. Cheap to lose again, so pinned.
+    renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff({ distance_m: 772.49 }) });
+
+    const tile = document.querySelector('[class*="admissionsTileDistance"]')!;
+    expect(tile.textContent).toMatch(/0\.48 miles\s+770 m/);
+    expect(tile.textContent).not.toMatch(/miles\d/);
+  });
+
   it('never shows the figure without saying it is not a catchment', () => {
     renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff() });
 

nextjs-app/__tests__/lib/site.test.ts

@@ -0,0 +1,27 @@
+import { SITE_URL, absoluteUrl } from '@/lib/site';
+
+describe('SITE_URL', () => {
+  it('is the www host, which is the one that serves a 200', () => {
+    // The apex 301s to www at Cloudflare. A canonical pointing at a redirect
+    // is a wasted signal, so every absolute URL we emit must already be www.
+    expect(SITE_URL).toBe('https://www.schoolcompare.co.uk');
+  });
+
+  it('has no trailing slash, so joins never double up', () => {
+    expect(SITE_URL.endsWith('/')).toBe(false);
+  });
+});
+
+describe('absoluteUrl', () => {
+  it('joins a rooted path', () => {
+    expect(absoluteUrl('/rankings')).toBe('https://www.schoolcompare.co.uk/rankings');
+  });
+
+  it('joins a path missing its leading slash', () => {
+    expect(absoluteUrl('rankings')).toBe('https://www.schoolcompare.co.uk/rankings');
+  });
+
+  it('maps the site root to a bare trailing slash', () => {
+    expect(absoluteUrl('/')).toBe('https://www.schoolcompare.co.uk/');
+  });
+});

nextjs-app/app/admissions/page.tsx

@@ -1,3 +1,4 @@
+import { absoluteUrl } from '@/lib/site';
 import type { Metadata } from 'next';
 import { AdmissionsView } from '@/components/AdmissionsView';
 
@@ -7,6 +8,7 @@ export const metadata: Metadata = {
   title: 'School Admissions Guide',
   description:
     'Understand the Primary and Secondary school admissions process in England, with live countdowns to every key deadline and National Offer Day.',
+  alternates: { canonical: absoluteUrl('/admissions') },
 };
 
 export default function AdmissionsPage() {

nextjs-app/app/compare/page.tsx

@@ -5,6 +5,7 @@
 
 import { fetchComparison, fetchMetrics } from '@/lib/api';
 import { ComparisonView } from '@/components/ComparisonView';
+import { absoluteUrl } from '@/lib/site';
 import type { Metadata } from 'next';
 
 interface ComparePageProps {
@@ -14,13 +15,33 @@ interface ComparePageProps {
   }>;
 }
 
-export const metadata: Metadata = {
+/**
-  title: 'Compare Schools',
+ * Indexability depends on the query string, so this cannot be a static export.
-  description:
+ *
-    'Compare schools in England side by side — Ofsted inspections, KS2 and GCSE results against the England average, admissions odds and school community.',
+ * Bare /compare is the landing page for the "compare schools" head term and
-  keywords:
+ * stays indexable. /compare?urns=… is an unbounded parameter space — 25,193
-    'school comparison, compare schools, Ofsted comparison, school admissions, KS2 comparison, primary school performance',
+ * schools make ~317 million pairs — so it goes noindex. It stays `follow` and
-};
+ * keeps a canonical to the bare path, so the links out to each school page
+ * still count.
+ */
+export async function generateMetadata(
+  { searchParams }: ComparePageProps,
+): Promise<Metadata> {
+  const { urns } = await searchParams;
+
+  const base: Metadata = {
+    title: 'Compare Schools',
+    description:
+      'Compare schools in England side by side — Ofsted inspections, KS2 and GCSE results against the England average, admissions odds and school community.',
+    keywords:
+      'school comparison, compare schools, Ofsted comparison, school admissions, KS2 comparison, primary school performance',
+    alternates: { canonical: absoluteUrl('/compare') },
+  };
+
+  if (!urns) return base;
+
+  return { ...base, robots: { index: false, follow: true } };
+}
 
 // Dynamic via searchParams; remove force-dynamic so internal data fetches
 // can still use Next.js's per-call revalidate cache.

nextjs-app/app/layout.tsx

@@ -5,6 +5,7 @@ import { Navigation } from '@/components/Navigation';
 import { Footer } from '@/components/Footer';
 import { ComparisonToast } from '@/components/ComparisonToast';
 import { ComparisonProvider } from '@/context/ComparisonProvider';
+import { SITE_URL } from '@/lib/site';
 import './globals.css';
 
 // Manrope carries headings and key messaging — the guideline's "friendly,
@@ -57,12 +58,12 @@ export const metadata: Metadata = {
   // No `icons` key on purpose: setting it here would override the file
   // conventions. app/icon.svg and app/apple-icon.tsx are the source, and
   // app/opengraph-image.tsx supplies og:image and twitter:image.
-  metadataBase: new URL('https://schoolcompare.co.uk'),
+  metadataBase: new URL(SITE_URL),
   openGraph: {
     type: 'website',
     title: 'schoolcompare | Compare School Performance',
     description: 'Compare primary and secondary school SATs and GCSE performance across England',
-    url: 'https://schoolcompare.co.uk',
+    url: SITE_URL,
     siteName: 'schoolcompare',
   },
   twitter: {

nextjs-app/app/page.tsx

@@ -3,6 +3,7 @@
  * Main landing page with school search and browsing
  */
 
+import { absoluteUrl } from '@/lib/site';
 import type { Metadata } from 'next';
 import { fetchSchools, fetchFilters, fetchDataInfo } from '@/lib/api';
 import { formatAcademicYear } from '@/lib/utils';
@@ -35,6 +36,10 @@ interface HomePageProps {
 export const metadata: Metadata = {
   title: { absolute: 'schoolcompare | Compare every school in England' },
   description: 'Search and compare school performance across England',
+  // This page reads eleven search params. They filter a result set; they do
+  // not make a new document. Collapsing every combination onto "/" stops the
+  // homepage competing with itself for its own head terms.
+  alternates: { canonical: absoluteUrl('/') },
 };
 
 // The page reads searchParams, which makes rendering dynamic by default.

nextjs-app/app/rankings/page.tsx

@@ -5,6 +5,7 @@
 
 import { fetchRankings, fetchFilters, fetchMetrics } from '@/lib/api';
 import { RankingsView } from '@/components/RankingsView';
+import { absoluteUrl } from '@/lib/site';
 import type { Metadata } from 'next';
 
 interface RankingsPageProps {
@@ -20,6 +21,9 @@ export const metadata: Metadata = {
   title: 'School Rankings',
   description: 'Top-ranked schools by SATs and GCSE performance across England',
   keywords: 'school rankings, top schools, best schools, KS2 rankings, KS4 rankings, school league tables',
+  // Param forms (?metric=&local_authority=&year=&phase=) collapse here for
+  // now. W3 replaces them with real indexable paths.
+  alternates: { canonical: absoluteUrl('/rankings') },
 };
 
 // Dynamic via searchParams; remove force-dynamic so internal data fetches

nextjs-app/app/robots.ts

@@ -4,6 +4,7 @@
  */
 
 import { MetadataRoute } from 'next';
+import { absoluteUrl } from '@/lib/site';
 
 export default function robots(): MetadataRoute.Robots {
   return {
@@ -14,6 +15,6 @@ export default function robots(): MetadataRoute.Robots {
         disallow: ['/api/', '/_next/'],
       },
     ],
-    sitemap: 'https://schoolcompare.co.uk/sitemap.xml',
+    sitemap: absoluteUrl('/sitemap.xml'),
   };
 }

nextjs-app/app/school/[slug]/page.tsx

@@ -15,6 +15,7 @@ import {
 } from '@/lib/schoolSections';
 import { parseSchoolSlug, schoolUrl } from '@/lib/utils';
 import type { NationalAverages } from '@/lib/types';
+import { absoluteUrl } from '@/lib/site';
 import type { Metadata } from 'next';
 
 /**
@@ -97,7 +98,7 @@ export async function generateMetadata({ params }: SchoolPageProps): Promise<Met
         title,
         description,
         type: 'website',
-        url: `https://schoolcompare.co.uk${canonicalPath}`,
+        url: absoluteUrl(canonicalPath),
         siteName: 'schoolcompare',
       },
       twitter: {
@@ -106,7 +107,7 @@ export async function generateMetadata({ params }: SchoolPageProps): Promise<Met
         description,
       },
       alternates: {
-        canonical: `https://schoolcompare.co.uk${canonicalPath}`,
+        canonical: absoluteUrl(canonicalPath),
       },
     };
   } catch {

nextjs-app/app/sitemap.xml/route.ts

@@ -1,32 +1,8 @@
-/**
+import { proxySitemap } from '@/lib/sitemapProxy';
- * Runtime proxy for /sitemap.xml → the FastAPI backend's generated sitemap.
- *
- * Like the /api/* proxy, this reads FASTAPI_URL at request time rather than
- * baking the backend host into the build, so one image works in every
- * environment. robots.ts points crawlers here.
- */
-
-import { NextResponse } from 'next/server';
 
 export const dynamic = 'force-dynamic';
 export const runtime = 'nodejs';
 
-function backendOrigin(): string {
-  const base = process.env.FASTAPI_URL || process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000/api';
-  return base.replace(/\/api$/, '');
-}
-
 export async function GET() {
-  let upstream: Response;
+  return proxySitemap('/sitemap.xml');
-  try {
-    upstream = await fetch(`${backendOrigin()}/sitemap.xml`, { cache: 'no-store' });
-  } catch {
-    return new NextResponse('Sitemap temporarily unavailable', { status: 502 });
-  }
-
-  const body = await upstream.text();
-  return new NextResponse(body, {
-    status: upstream.status,
-    headers: { 'content-type': upstream.headers.get('content-type') || 'application/xml' },
-  });
 }

nextjs-app/app/sitemaps/[...parts]/route.ts

@@ -0,0 +1,24 @@
+import { NextResponse } from 'next/server';
+import { proxySitemap } from '@/lib/sitemapProxy';
+
+export const dynamic = 'force-dynamic';
+export const runtime = 'nodejs';
+
+/**
+ * Children are /sitemaps/static.xml and /sitemaps/schools-{n}.xml. The name is
+ * validated here rather than passed through, so this route cannot be used to
+ * reach arbitrary backend paths.
+ */
+const CHILD = /^(static|schools-\d+)\.xml$/;
+
+export async function GET(
+  _request: Request,
+  { params }: { params: Promise<{ parts: string[] }> },
+) {
+  const { parts } = await params;
+  const name = parts.join('/');
+  if (!CHILD.test(name)) {
+    return new NextResponse('Not found', { status: 404 });
+  }
+  return proxySitemap(`/sitemaps/${name}`);
+}

nextjs-app/components/school/AdmissionsSection.tsx

@@ -70,10 +70,16 @@ export function AdmissionsSection({
   {/* Spans both columns rather than taking a half-width cell. This is the
       figure parents come to the page for, and at tile width the two-line
       "0.31 miles / September 2025" pairing wraps badly. */}
+  {/* The {' '} between the figure and its metric support is not decoration.
+      The two are flex children, so the gap is drawn by CSS and the text layer
+      had nothing between them: textContent read "0.88 miles1.4 km", which is
+      what a screen reader announces and what any text matcher sees. Whitespace
+      text nodes are not rendered as flex items, so this changes the reading
+      without changing the layout. */}
   const distanceTile = cutoff && (
     <div className={`${styles.admissionsTile} ${styles.admissionsTileDistance}`}>
       <dd className={styles.admissionsTileNum}>
-        {cutoff.primary}
+        {cutoff.primary}{' '}
         <span className={styles.admissionsTileSub}>{cutoff.secondary}</span>
       </dd>
       <dt className={styles.admissionsTileLabel}>

nextjs-app/components/school/SecondaryAdmissionsSection.tsx

@@ -79,7 +79,7 @@ export function SecondaryAdmissionsSection({
               Last distance offered · {cutoff.entryYear}
             </div>
             <div className={styles.metricValue}>
-              {cutoff.primary}
+              {cutoff.primary}{' '}
               <span className={styles.metricValueSub}>{cutoff.secondary}</span>
             </div>
           </div>

nextjs-app/lib/site.ts

@@ -0,0 +1,19 @@
+/**
+ * The one place the site's absolute origin is written down.
+ *
+ * The apex domain 301s to www at Cloudflare, so www is the host that actually
+ * serves a 200. Canonicals, og:url, sitemap <loc> entries and the robots.txt
+ * Sitemap: line must all agree with it — a canonical pointing at a redirect
+ * makes Google resolve the hop before it can consolidate the signal.
+ *
+ * backend/app.py holds the same value as BASE_URL for the sitemap. The two are
+ * asserted against each other by the e2e journeys rather than shared at build
+ * time, because the backend and frontend ship as separate images.
+ */
+export const SITE_URL = 'https://www.schoolcompare.co.uk';
+
+/** Absolute URL for a site-relative path. Tolerates a missing leading slash. */
+export function absoluteUrl(path: string): string {
+  const rooted = path.startsWith('/') ? path : `/${path}`;
+  return `${SITE_URL}${rooted}`;
+}

nextjs-app/lib/sitemapProxy.ts

@@ -0,0 +1,29 @@
+/**
+ * Runtime proxy for the sitemap family → the FastAPI backend.
+ *
+ * Like the /api/* proxy, this reads FASTAPI_URL at request time rather than
+ * baking the backend host into the build, so one image works in every
+ * environment. robots.ts points crawlers at /sitemap.xml, which is the index;
+ * the index names children under /sitemaps/, which land on the same proxy.
+ */
+import { NextResponse } from 'next/server';
+
+function backendOrigin(): string {
+  const base = process.env.FASTAPI_URL || process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000/api';
+  return base.replace(/\/api$/, '');
+}
+
+export async function proxySitemap(path: string): Promise<NextResponse> {
+  let upstream: Response;
+  try {
+    upstream = await fetch(`${backendOrigin()}${path}`, { cache: 'no-store' });
+  } catch {
+    return new NextResponse('Sitemap temporarily unavailable', { status: 502 });
+  }
+
+  const body = await upstream.text();
+  return new NextResponse(body, {
+    status: upstream.status,
+    headers: { 'content-type': upstream.headers.get('content-type') || 'application/xml' },
+  });
+}

pipeline/transform/dbt_project.yml

@@ -24,6 +24,19 @@ clean-targets: ["target", "dbt_packages"]
 vars:
   non_england_school_type_codes: [25, 26, 30, 37]
 
+  # Type alone is not enough. Beechwood College (URN 142458, Sully, CF64 5SE)
+  # is a Welsh school that GIAS types as a Special post 16 institution (32)
+  # rather than a Welsh establishment (30), so the type filter let it through.
+  # LA code is the reliable discriminator: GIAS gives the 22 Welsh unitary
+  # authorities the contiguous block 660-681, which English authorities never
+  # use (they occupy 201-384 and 800-938).
+  #
+  # Filtering on postcode would be wrong: Redbrook, Tutshill, Wyedean and two
+  # other Gloucestershire schools carry NP16/NP25 postcodes because Royal Mail
+  # areas straddle the border, and they are English schools with English data.
+  welsh_la_code_min: 660
+  welsh_la_code_max: 681
+
 models:
   school_compare:
     staging:

pipeline/transform/models/marts/dim_location.sql

@@ -37,3 +37,5 @@ where s.status_code in (1, 3)
 -- coalesce, not a bare NOT IN: a null type code would make the predicate
 -- null and drop the row silently. Unknown type is not grounds for exclusion.
 and coalesce(s.school_type_code, -1) not in ({{ var('non_england_school_type_codes') | join(', ') }})
+-- Welsh schools GIAS does not type as Welsh establishments (see vars).
+and coalesce(s.local_authority_code, -1) not between {{ var('welsh_la_code_min') }} and {{ var('welsh_la_code_max') }}

pipeline/transform/models/marts/dim_school.sql

@@ -96,3 +96,5 @@ where s.status_code in (1, 3)
 -- coalesce, not a bare NOT IN: a null type code would make the predicate
 -- null and drop the row silently. Unknown type is not grounds for exclusion.
 and coalesce(s.school_type_code, -1) not in ({{ var('non_england_school_type_codes') | join(', ') }})
+-- Welsh schools GIAS does not type as Welsh establishments (see vars).
+and coalesce(s.local_authority_code, -1) not between {{ var('welsh_la_code_min') }} and {{ var('welsh_la_code_max') }}

pipeline/transform/tests/assert_england_only_schools.sql

@@ -26,3 +26,29 @@ select
 from {{ ref('dim_location') }} l
 left join {{ ref('dim_school') }} s on l.urn = s.urn
 where s.urn is null
+
+union all
+
+-- Belt and braces on the Welsh LA code range.
+--
+-- The range 660-681 is documented GIAS structure, not something this project
+-- can verify from its own data — local_authority_code is not exposed anywhere
+-- the app can assert on. So the range does the filtering and the authority
+-- NAME does the checking: if 660-681 is ever wrong or incomplete, a Welsh
+-- authority reappears here and the pipeline fails loudly rather than the
+-- corpus quietly regrowing a Welsh tail.
+--
+-- Beechwood College (142458) is why this exists: a Welsh school GIAS types as
+-- a Special post 16 institution, which the type filter alone let through.
+select
+    l.urn,
+    l.local_authority_name as school_name,
+    null::integer          as school_type_code
+from {{ ref('dim_location') }} l
+where l.local_authority_name in (
+    'Isle of Anglesey', 'Gwynedd', 'Conwy', 'Denbighshire', 'Flintshire',
+    'Wrexham', 'Powys', 'Ceredigion', 'Pembrokeshire', 'Carmarthenshire',
+    'Swansea', 'Neath Port Talbot', 'Bridgend', 'Vale of Glamorgan',
+    'Rhondda Cynon Taf', 'Merthyr Tydfil', 'Caerphilly', 'Blaenau Gwent',
+    'Torfaen', 'Monmouthshire', 'Newport', 'Cardiff'
+)