Merge pull request 'fix(seo): crawl hygiene and a per-family sitemap index (W1)' (#108) from feat/seo-crawl-hygiene into feat/england-only-corpus

Reviewed-on: #108

backend/app.py

@@ -79,12 +79,6 @@ def _school_url(urn: int, school_name: str) -> str:
 STATIC_SITEMAP_PATHS = ("/", "/rankings", "/compare", "/admissions")
 
 
-# A page has something a search result could state if any of these is present
-# in any year. Shared by _has_publishable_data and the per-school check in
-# _school_sitemap_rows so the two can never drift.
-_PUBLISHABLE_FIELDS = ("rwm_expected_pct", "attainment_8_score", "ofsted_grade")
-
-
 def _has_publishable_data(row) -> bool:
     """True when a school page has something a search result could state.
 
@@ -93,7 +87,7 @@ def _has_publishable_data(row) -> bool:
     signal down, so it stays out of the sitemap. The page itself still resolves
     for anyone who has the URL.
     """
-    for field in _PUBLISHABLE_FIELDS:
+    for field in ("rwm_expected_pct", "attainment_8_score", "ofsted_grade"):
         value = row.get(field)
         if value is not None and not pd.isna(value):
             return True
@@ -121,21 +115,6 @@ def _school_sitemap_rows(df) -> list[str]:
     rows: list[str] = []
     seen: set[int] = set()
 
-    # Publishable is a property of the SCHOOL, not of its latest row.
-    #
-    # The first cut tested the latest year's row alone, which quietly dropped
-    # every school that has results in its history but a null row for the most
-    # recent year — a school that stopped reporting, or whose figures were
-    # suppressed for small-cohort disclosure. The Mallard Academy (150367) is
-    # the case that caught it: real KS2 results for 2015-16 through 2018-19,
-    # then null rows for 2022-23 onward. Its page shows all four years; the
-    # sitemap omitted it. Roughly 220 schools were affected.
-    publishable_cols = [c for c in _PUBLISHABLE_FIELDS if c in df.columns]
-    publishable: set[int] = (
-        set(df.loc[df[publishable_cols].notna().any(axis=1), "urn"].astype(int))
-        if publishable_cols else set()
-    )
-
     # Latest row per URN first, so a school's most recent Ofsted date wins.
     ordered = df.sort_values("year", ascending=False) if "year" in df.columns else df
 
@@ -144,7 +123,7 @@ def _school_sitemap_rows(df) -> list[str]:
         if urn in seen:
             continue
         seen.add(urn)
-        if urn not in publishable:
+        if not _has_publishable_data(row):
             continue
 
         lastmod = None

backend/tests/test_sitemap.py

@@ -176,44 +176,3 @@ def test_children_are_chunked_under_the_limit(monkeypatch):
 def test_build_sitemap_still_returns_the_index(sitemap):
     # lifespan and the admin endpoint call build_sitemap(); keep it working.
     assert "<sitemapindex" in sitemap
-
-
-def test_school_with_results_in_an_earlier_year_is_still_listed(monkeypatch):
-    """Regression: The Mallard Academy (150367).
-
-    Real KS2 results 2015-16 to 2018-19, then null rows from 2022-23 onward
-    because the school stopped reporting. The first cut tested the latest
-    year's row alone and dropped it, along with ~220 others, even though its
-    detail page shows all four years of results.
-    """
-    from backend import app as app_module
-    import pandas as _pd
-
-    base = {"local_authority": "Testshire", "school_type": "Academy",
-            "phase": "Primary", "ofsted_date": None, "ofsted_grade": np.nan,
-            "attainment_8_score": np.nan, "urn": 150367,
-            "school_name": "Mallard Academy"}
-    df = _pd.DataFrame([
-        {**base, "year": 201819, "rwm_expected_pct": 67.0},
-        {**base, "year": 202324, "rwm_expected_pct": np.nan},
-        {**base, "year": 202425, "rwm_expected_pct": np.nan},
-    ])
-    monkeypatch.setattr(app_module, "load_school_data", lambda: df)
-
-    xml = app_module.build_sitemaps()["schools-1.xml"]
-    assert "/school/150367-mallard-academy" in xml
-
-
-def test_school_with_no_results_in_any_year_is_still_omitted(monkeypatch):
-    """The fix must not turn into "list everything"."""
-    from backend import app as app_module
-    import pandas as _pd
-
-    base = {"local_authority": "Testshire", "school_type": "Academy",
-            "phase": "Primary", "ofsted_date": None, "ofsted_grade": np.nan,
-            "attainment_8_score": np.nan, "rwm_expected_pct": np.nan,
-            "urn": 100002, "school_name": "Ghost Primary"}
-    df = _pd.DataFrame([{**base, "year": y} for y in (202324, 202425)])
-    monkeypatch.setattr(app_module, "load_school_data", lambda: df)
-
-    assert "/school/100002" not in app_module.build_sitemaps()["schools-1.xml"]

docs/branch-cleanup-2026-08-20.txt

@@ -1,85 +0,0 @@
-# Remote branch cleanup, 2026-08-20
-# Restore any branch with:  git push origin <sha>:refs/heads/<name>
-
-## Deleted: fully merged into main (content is in main)
-75677f4252b759ef895e7d5f7c19f8f1745bdb59 add-contact-form-footer
-fa1abff642683dfd26ba88a295a0a6710d77147a chore/byline-removal-and-audit-figure
-95081d38bdf87764ef5d298676c25fae4cd3b792 chore/remove-parent-view
-6877abedebfc1c7d95f1f6ebe68945c62be528ab chore/staged-prod-promotion
-090d5f7bec824e083d3252e2c6e636686016304d ci/frontend-checks-speedup
-8c3a5cc4e9f551f0190d85357ce7741ad87f3a4c design/cohort-identity
-955659580067ca8b81bd77e31e4e2554103f094d feat/allow-analytics-iframe-embed
-6828f6cd4417284ea3eb6f088fa20945b8b40ed3 feat/compare-chips-two-per-row
-d5cd0abfee226885119665da5d2aa8288b59217f feat/compare-data-foundation
-6dd9b04b50bee146682da87efad8fc8b526251c5 feat/compare-frontend-rebuild
-96d5fcf5b07b6f175b48e9b20fcb765a320a907f feat/detail-header-details-reveal
-f1388ff5bd0af1409823a1e047b7ba84246a0f70 feat/gias-sixth-form-flag
-eddf74745f86c9c6d9eb07d867246ff9cc90dc20 feat/hero-artwork-v2
-3015c37bac6dc28db58c80fdb9235942c25b83d7 feat/hero-byline
-8e763e39d17a4964cf558e51c03f044186371f6d feat/info-popover-tooltip
-88c653215d520ab6e902c9de55bf27d86eeb90c3 feat/last-distance-offered
-a72323874f7aebdb2e64b6d64a5febd61152d09d feat/last-distance-offered-full
-c9a1892bfb0370e0672e5849cba294ddfabe3c65 feat/latest-cutoff-only
-4e8df006d75d8be2a1d8529ddf855c445854cba1 feat/near-me-by-search
-45ab479062c6a1639facad636fc0cc0cf0fd9155 feat/proposed-to-close-schools
-3bf2e8f262cbe058fda6de6f8ea3e050224a51b9 feat/school-detail-visualisations
-1f80571b1ff217dc92b660a936c02b5f49d07f0b feat/umami-heatmap-recorder
-609bb923d96aa5730131463ea35d5efdd404bf96 feature/ingest-independent-schools
-94151c58ea38a9256d66d15161293486a500c7a2 fix/admissions-section-height
-79246edc22961c2beb3520437e9d064d3b809d10 fix/annual-dag-ks4-national-selector
-59ac9c10b97e0ef1143f57fea06b324e72ac3d4a fix/chart-marker-contrast
-9f8dba227c95706ca3527bd48d381e7622cc0a5e fix/compare-chart-refetch-resilience
-e74d3882ce78a141fa1a57daa3102d7a58852dc3 fix/compare-expert-fixes
-80176cac4db4820e76ea2a156c7a2974bee2f203 fix/compare-final-review-mustfix
-f579630fab6c456e26a6984a3e8eebdbe3184202 fix/compare-mockup-drift
-dc85254ad2ddf134b4434065d4762cef374d2220 fix/compare-null-year-blanks-chart
-43a2c4a6bc539b621f31655aec05ef319a25f343 fix/compare-refresh-and-fetch
-d677b5453365b72c81d6df2de62b1fa0d05d684d fix/daily-dag-cache-invalidation
-f6bb037c471553e8195b5a8b147467ce0d07a688 fix/detail-all-through
-17bd4d5a5eb0b12ca79b97db587f14f7d671e85b fix/detail-chart-truthfulness
-4e6be0ce65647410b4ff74f8b763207c28920c26 fix/detail-inclusion-admissions
-fdda52ff0af3fae03a4b059a655973cdbb269f91 fix/detail-ofsted-correctness
-e36125b24aba254a8d15c5c33b4d2a296e691995 fix/detail-provenance-anchoring
-b31e71ac884df6507f567adc046c9fc52d9d310a fix/detail-report-card-render-date
-32f8a02862be6a1d49f4c3928b17fcf15d4c94cc fix/detail-trend-chart-taller
-e65688d600a86818fe21ae4c61ba27e5b6ec8d7c fix/e2e-brand-assertions
-3adea73ee04cdedfab54b0351878b297f72756ad fix/e2e-compare-chips-phase
-06e4898c30feaedc471f97aba28ddb0d61379f4d fix/e2e-compare-samephase
-9abd020967670a855e80fe5a908c8048a3aa9f14 fix/e2e-distance-locator
-acec8135e1ec7c9c3c255e5b23733a0ef862b590 fix/e2e-rankings-year-pick
-5944d88f0b1517ef1ef56af1b62270d2cf28e717 fix/expert-signoff-mustfixes
-2433101fa08be5df6f170d41790512ffe823d33e fix/font-cascade-and-map-palette
-74ca76d150deec6725259d9637ea86d7bb90c683 fix/gias-legacy-fallback
-bdaa05cd542f563ef74c45307cd8f7fc465193c9 fix/hero-fallback-and-sharp
-d52d384cf23d282b44e9251176f8f3402d808600 fix/hero-map-ios-fullscreen
-4043270a77bbe4edb18207fa5f1d94d4747fe12f fix/hero-mobile-and-wording
-22e9eb2d48b0d6623e88fd67cb6ca8e5e4074583 fix/homepage-education-accuracy
-8d50afef1e8a2b621b7344eadf475b0d609ad7a2 fix/leaflet-specificity-and-font-assertion
-b2b2cad5acf534ae7a667d3fb2be15efff37c4a7 fix/list-map-report-card-signal
-dc21e80a5e9eebd13aaab84735642eb77cef35e6 fix/mobile-cell-name-size
-e5f7f4c959f024c073472122d858333a0f24866c fix/mobile-compare-polish
-a00cbe916182d1e04661c750a1ce2e7ad27a68ae fix/mobile-sort-select-overflow
-2fd997bfe640c419a6713e85df008463de7f56e8 fix/modal-keyboard-viewport
-3e7705756776a0c44d27966dfc972023f1f38b69 fix/ofsted-link-text
-ce422e64363e2b03c186ef316832b6de15502d67 fix/promote-status-token
-4522cbf64560db1e1cd519e119aa466b42cd16a1 fix/proposed-to-close-copy
-15da060e4af37fbae919e0edf25f108266de2585 fix/rankings-admissions-accuracy
-6c872ce726f210433354ca38dc5314bc6a467534 fix/rankings-year-validation
-1c1df7796194af3d47f8e5ac0a0fbe6f323700e7 fix/report-card-chip-alignment
-b2dc4d0779ced02709429afaa5985acf4abda794 fix/results-map-ios-fullscreen
-95a5783da1fc994df76cb97238b55596dce4cd8f fix/runtime-api-proxy
-8a9ba30cc24e29653a72c03c0e817684b7db7c07 fix/sats-per-level-national
-536832a524fc4f9ed858f047e00b9a4d9d429c46 fix/school-detail-nan-500
-fef83b3bf244a9bf3cb4afa75dbc433d8725014f fix/schoolbar-sticky-offset
-b0c5b6bb57c879477da12c37b15cff950c29ebd3 fix/secondary-anchors-button-affordance
-261403bcd2b01aa4f26ee212e26302fc0f769bf9 fix/special-note-full-width
-ea5249a2ea6faf6bfa5a1522644387ba59770ec8 fix/standardise-distance-units
-e4565e9f158721d4df6b918f2b065de82851f8d9 fix/trends-chart-height
-3aad5101a842539105022f9059e85c224fc5973a fix/welsh-establishment-leak
-315f1feede70bdf3101d2fdd305b3d06e037fdac perf/batch-supplementary
-d2dc78aeb599e16b7ef5019be2b360b08df463bc perf/compare-loading
-e098ad4bd1130152705788d4773837b7d1e7112e perf/server-client-split
-
-## Deleted: superseded by PR #110 (content preserved on feat/seo-crawl-hygiene-main)
-786ec80dd4de4a3cb674a89e35b3b5e461639289 feat/seo-crawl-hygiene
-a5ac0bcd1b37bc10dcbce88f8601d01bf7b3eaaf feat/england-only-corpus

e2e/tests/journeys.spec.ts

@@ -1252,11 +1252,7 @@ test('a published cut-off distance is shown with the year it belongs to', async
   await expect(label).toContainText(`September ${found!.distance.year}`);
 
   // And the figure itself, in the unit councils publish in.
-  // No trailing \b: the tile's support figure is an adjacent text node, so
-  // the element reads "0.88 miles 1.4 km" and a word boundary after "miles"
-  // is not guaranteed. The leading shape is what matters — a decimal figure
-  // in miles, never a metric one.
-  await expect(page.getByText(/\d+\.\d+ miles/).first()).toBeVisible();
+  await expect(page.getByText(/\d+(\.\d+)? (miles|m)\b/).first()).toBeVisible();
 });
 
 test('a cut-off distance is never shown without saying it is not a catchment', async ({ page }) => {
@@ -1344,27 +1340,7 @@ test('a school page shows no year-by-year cut-off record', async ({ page }) => {
   // history back without the API.
   await expect(page.getByText(/Last distance offered, by year/)).toHaveCount(0);
   await expect(page.getByText(/too few to read as a trend/)).toHaveCount(0);
-
-  /*
-   * No "Distance" tab in the admissions segmented control.
-   *
-   * Scoped to the control, and exact, because getByRole matches accessible
-   * names by case-insensitive SUBSTRING: an unscoped { name: 'Distance' }
-   * matched "Show this distance on a map" — the map toggle added by this same
-   * feature — and failed a page that was entirely correct. Naming the group
-   * this assertion is about also means unrelated copy elsewhere on the page
-   * can never break it again.
-   */
-  const viewToggle = page.getByRole('group', { name: 'Admissions view' });
-  const tabs = await viewToggle.getByRole('button').allTextContents();
-
-  // Read the tabs positively rather than asserting an absence against a
-  // locator that might resolve to nothing: if the group selector ever stops
-  // matching, an absence check passes for the wrong reason, which is how the
-  // bug this test is guarding would slip back in unnoticed.
-  expect(tabs, 'admissions view toggle did not resolve').toContain('This year');
-  expect(tabs.map((s) => s.trim()), `admissions tabs: ${tabs.join(', ')}`)
-    .not.toContain('Distance');
+  await expect(page.getByRole('button', { name: 'Distance' })).toHaveCount(0);
 });
 
 test('the postcode check answers for the published year, and names it', async ({ page }) => {
@@ -1378,7 +1354,7 @@ test('the postcode check answers for the published year, and names it', async ({
   await expect(section).toBeVisible({ timeout: 15_000 });
 
   await page.getByLabel('Your postcode').fill('SW1A 1AA');
-  await page.getByRole('button', { name: 'Check', exact: true }).click();
+  await page.getByRole('button', { name: 'Check' }).click();
 
   const result = page.getByRole('status');
   const error = page.getByRole('alert');
@@ -1545,32 +1521,6 @@ test('searching a Welsh authority by name returns no schools', async ({ page })
   expect(body.total ?? (body.schools ?? []).length).toBe(0);
 });
 
-test('a Welsh school GIAS does not type as Welsh is gone too', async ({ page }) => {
-  // Beechwood College, Sully, CF64 5SE. GIAS types it "Special post 16
-  // institution" (32), not "Welsh establishment" (30), so the establishment
-  // type filter alone left it behind — it was the last Welsh school standing.
-  const res = await page.request.get('/api/schools?local_authority=Vale%20of%20Glamorgan&page_size=1');
-  expect(res.ok()).toBeTruthy();
-  const body = await res.json();
-  expect(body.total ?? (body.schools ?? []).length).toBe(0);
-
-  const page404 = await page.goto('/school/142458-beechwood-college');
-  expect(page404?.status()).toBe(404);
-});
-
-test('English schools with Welsh postcodes are kept', async ({ page }) => {
-  // Royal Mail postcode areas straddle the border: these Gloucestershire
-  // schools carry NP16/NP25 postcodes but are English schools with English
-  // data. A postcode-based filter would have wrongly deleted them, so this
-  // guards the fix from being "simplified" into one.
-  for (const urn of [115639, 115655, 137382]) {
-    const res = await page.request.get(`/api/schools/${urn}`);
-    expect(res.ok(), `URN ${urn} should still resolve`).toBeTruthy();
-    const { school_info } = await res.json();
-    expect(school_info.local_authority).toBe('Gloucestershire');
-  }
-});
-
 test('a Welsh school URL 404s while an English one still resolves', async ({ page }) => {
   // Paired on purpose: the Welsh assertion alone would also pass if the whole
   // site were down, which is the failure this test most needs to distinguish.
@@ -1695,66 +1645,3 @@ test('a bare /compare is indexable, a parameterised one is not', async ({ page }
     .getAttribute('href');
   expect(canonical).toBe('https://www.schoolcompare.co.uk/compare');
 });
-
-/*
- * Staging must not be indexable (spec 2026-08-20, W1 hygiene).
- *
- * These journeys only ever run against staging — deploy.yml passes
- * STAGING_BASE_URL, and promote.yml only smoke-polls production without
- * Playwright — so asserting the noindex header here is safe.
- */
-test('staging answers noindex, and stays crawlable so the noindex is seen', async ({ page }) => {
-  const res = await page.request.get('/');
-  expect(res.ok()).toBeTruthy();
-
-  const tag = res.headers()['x-robots-tag'];
-  expect(tag, 'staging must send X-Robots-Tag').toBeTruthy();
-  expect(tag).toContain('noindex');
-
-  // The other half, and the reason this is one test rather than two: a
-  // Disallow would stop Google fetching the page at all, so it would never
-  // see the noindex above. The two only work together.
-  const robots = await (await page.request.get('/robots.txt')).text();
-  expect(robots).not.toMatch(/^\s*Disallow:\s*\/\s*$/mi);
-});
-
-test('a school page on staging is noindexed too, not just the homepage', async ({ page }) => {
-  const list = await page.request.get('/api/schools?search=primary&per_page=1');
-  const [first] = (await list.json()).schools ?? [];
-  expect(first, 'no school available').toBeTruthy();
-
-  const res = await page.request.get(`/school/${first.urn}-x`);
-  expect(res.headers()['x-robots-tag']).toContain('noindex');
-});
-
-/*
- * W8 — the C1 pages must ship a description, and it must differentiate.
- *
- * Baseline was 0.43% CTR at position 6.1 on "compare school performance",
- * against 9.16% for the brand query from the same neighbourhood. The SERP is
- * owned by the DfE's own service, so a description that paraphrases it earns
- * nothing. Google may rewrite a snippet, but it cannot use one we never sent.
- */
-test('every C1 page ships a description, and none opens its title with the brand', async ({ page }) => {
-  for (const path of ['/', '/compare', '/rankings', '/admissions']) {
-    await page.goto(path);
-
-    const desc = await page.locator('meta[name="description"]').first()
-      .getAttribute('content');
-    expect(desc, `${path} must ship a description`).toBeTruthy();
-    expect(desc!.length, `${path} description too short to be worth reading`)
-      .toBeGreaterThan(100);
-
-    const title = await page.title();
-    expect(title.toLowerCase().startsWith('schoolcompare'),
-      `${path} spends its most valuable pixels on the brand`).toBe(false);
-  }
-});
-
-test('the homepage snippet names what gov.uk does not publish', async ({ page }) => {
-  await page.goto('/');
-  const desc = await page.locator('meta[name="description"]').first()
-    .getAttribute('content');
-  // Admissions distance is the one fact the DfE service has no equivalent for.
-  expect(desc).toMatch(/close you had to live|distance/i);
-});

nextjs-app/__tests__/app/metadata.test.ts

@@ -51,80 +51,3 @@ describe('/compare indexability', () => {
       .toBe('https://www.schoolcompare.co.uk/compare');
   });
 });
-
-/*
- * W8 — snippet copy for the C1 cluster.
- *
- * The baseline (GSC, 16 months to 2026-08-20) showed these pages ranking on
- * page one and converting at a tenth of the normal rate: "compare school
- * performance" at position 6.1 with 0.43% CTR, against 9.16% for the brand
- * query from the same neighbourhood. The SERP is dominated by the DfE's own
- * "Compare school performance" service, so the job of this copy is to say
- * what that service does not offer, without losing intent match on the title.
- *
- * These tests guard the mechanics that make a snippet work — length, intent
- * keyword, differentiator, no brand-first — not the exact wording, which
- * should stay free to iterate.
- */
-
-// Google truncates titles near 60 characters and descriptions near 155.
-const TITLE_MAX = 60;
-const DESC_MIN = 110;
-const DESC_MAX = 155;
-
-type Meta = { title?: unknown; description?: unknown };
-const titleOf = (m: Meta): string => {
-  const t = m.title as string | { absolute?: string } | undefined;
-  return typeof t === 'string' ? t : (t?.absolute ?? '');
-};
-
-describe('C1 snippet copy', () => {
-  const pages: Array<[string, Meta, RegExp]> = [
-    ['home', homeMetadata as Meta, /compare schools/i],
-    ['rankings', rankingsMetadata as Meta, /league table/i],
-    ['admissions', admissionsMetadata as Meta, /admission/i],
-  ];
-
-  for (const [name, meta, intent] of pages) {
-    it(`${name}: title carries the search intent and fits the SERP`, () => {
-      const t = titleOf(meta);
-      expect(t).toMatch(intent);
-      expect(t.length).toBeLessThanOrEqual(TITLE_MAX);
-    });
-
-    it(`${name}: title does not open with the brand`, () => {
-      // The measured 0.43% CTR came from a brand-first title. The most
-      // valuable pixels go to the thing the searcher typed.
-      expect(titleOf(meta).toLowerCase().startsWith('schoolcompare')).toBe(false);
-    });
-
-    it(`${name}: description is long enough to be worth reading, short enough to survive`, () => {
-      const d = meta.description as string;
-      expect(d.length).toBeGreaterThanOrEqual(DESC_MIN);
-      expect(d.length).toBeLessThanOrEqual(DESC_MAX);
-    });
-  }
-
-  it('the homepage description names what gov.uk does not publish', () => {
-    // Admissions distance is the one fact the DfE service has no equivalent
-    // for. If it ever leaves this description, the snippet is competing with
-    // gov.uk on gov.uk's own ground.
-    expect(homeMetadata.description).toMatch(/close you had to live|distance/i);
-  });
-
-  it('/compare targets the tool phrasing rather than repeating the homepage', () => {
-    // Two pages chasing one phrase is how a site competes with itself.
-    return compareMetadata({ searchParams: Promise.resolve({}) }).then((m) => {
-      expect(m.title).toMatch(/comparison tool/i);
-      expect(m.title).not.toBe(titleOf(homeMetadata as Meta));
-    });
-  });
-
-  it('no C1 page claims a school count that will drift', () => {
-    // The corpus moves with every data refresh; this repo has already shipped
-    // one copy bug of that kind ("three schools" against MAX_SCHOOLS = 5).
-    for (const [, meta] of pages) {
-      expect(meta.description as string).not.toMatch(/\b\d{2},\d{3}\b|\b\d{2},000\b/);
-    }
-  });
-});

nextjs-app/__tests__/components/lastDistanceOffered.test.tsx

@@ -30,17 +30,6 @@ describe('primary detail page', () => {
     expect(screen.getByText(/Last distance offered/)).toHaveTextContent('September 2024');
   });
 
-  it('keeps the figure and its metric support readable as two numbers', () => {
-    // They are flex children with a CSS gap and nothing between them in the
-    // text layer, which read as "0.48 miles770 m" to a screen reader and to any
-    // text matcher. Cheap to lose again, so pinned.
-    renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff({ distance_m: 772.49 }) });
-
-    const tile = document.querySelector('[class*="admissionsTileDistance"]')!;
-    expect(tile.textContent).toMatch(/0\.48 miles\s+770 m/);
-    expect(tile.textContent).not.toMatch(/miles\d/);
-  });
-
   it('never shows the figure without saying it is not a catchment', () => {
     renderSchoolDetail({ ...primaryFixture, admissionDistance: cutoff() });
 

nextjs-app/app/admissions/page.tsx

@@ -5,11 +5,9 @@ import { AdmissionsView } from '@/components/AdmissionsView';
 export const dynamic = 'force-static';
 
 export const metadata: Metadata = {
-  // Deadlines and offer days are what gets searched, and what this page is
-  // genuinely best at — the countdowns are live.
-  title: { absolute: 'School Admissions Deadlines & Offer Days | schoolcompare' },
+  title: 'School Admissions Guide',
   description:
-    'Every key date for primary and secondary school admissions in England, with live countdowns to the application deadline and National Offer Day.',
+    'Understand the Primary and Secondary school admissions process in England, with live countdowns to every key deadline and National Offer Day.',
   alternates: { canonical: absoluteUrl('/admissions') },
 };
 

nextjs-app/app/compare/page.tsx

@@ -30,12 +30,9 @@ export async function generateMetadata(
   const { urns } = await searchParams;
 
   const base: Metadata = {
-    // Deliberately not the homepage's phrase. Two pages chasing "compare
-    // schools" is how a site competes with itself; this one takes the tool
-    // phrasing instead.
-    title: 'School Comparison Tool — Up to Five at Once | schoolcompare',
+    title: 'Compare Schools',
     description:
-      'Put up to five English schools in one table: SATs and GCSE results against the England average, Ofsted grades, and the distance places were offered.',
+      'Compare schools in England side by side — Ofsted inspections, KS2 and GCSE results against the England average, admissions odds and school community.',
     keywords:
       'school comparison, compare schools, Ofsted comparison, school admissions, KS2 comparison, primary school performance',
     alternates: { canonical: absoluteUrl('/compare') },

nextjs-app/app/layout.tsx

@@ -48,11 +48,10 @@ export const metadata: Metadata = {
     statusBarStyle: 'default',
   },
   title: {
-    default: 'Compare Schools Side by Side | schoolcompare',
+    default: 'schoolcompare | Compare School Performance',
     template: '%s | schoolcompare',
   },
-  description:
-    'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place. Free, no sign-up.',
+  description: 'Compare primary and secondary school SATs and GCSE performance across England',
   keywords: 'school comparison, KS2 results, KS4 results, primary school, secondary school, England schools, SATs results, GCSE results',
   authors: [{ name: 'schoolcompare' }],
   manifest: '/manifest.json',
@@ -62,18 +61,16 @@ export const metadata: Metadata = {
   metadataBase: new URL(SITE_URL),
   openGraph: {
     type: 'website',
-    title: 'Compare Schools Side by Side | schoolcompare',
-    description:
-      'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place.',
+    title: 'schoolcompare | Compare School Performance',
+    description: 'Compare primary and secondary school SATs and GCSE performance across England',
     url: SITE_URL,
     siteName: 'schoolcompare',
   },
   twitter: {
     // summary_large_image now that there is an image worth showing.
     card: 'summary_large_image',
-    title: 'Compare Schools Side by Side | schoolcompare',
-    description:
-      'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place.',
+    title: 'schoolcompare | Compare School Performance',
+    description: 'Compare primary and secondary school SATs and GCSE performance across England',
   },
 };
 

nextjs-app/app/page.tsx

@@ -34,21 +34,8 @@ interface HomePageProps {
  * saying the brand twice.
  */
 export const metadata: Metadata = {
-  /*
-   * Intent in the title, differentiator in the description.
-   *
-   * These queries are owned by the DfE's own "Compare school performance"
-   * service, and the old title — brand first, then a near-paraphrase of that
-   * service's name — gave a searcher no reason to pick us over it. It drew
-   * 0.43% CTR at position 6.1 while the brand query drew 9.16% from the same
-   * neighbourhood, so the ranking was never the problem.
-   *
-   * The title now matches what people type. The description carries the one
-   * fact gov.uk does not publish: how close you had to live to get a place.
-   */
-  title: { absolute: 'Compare Schools Side by Side | schoolcompare' },
-  description:
-    'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place. Free, no sign-up.',
+  title: { absolute: 'schoolcompare | Compare every school in England' },
+  description: 'Search and compare school performance across England',
   // This page reads eleven search params. They filter a result set; they do
   // not make a new document. Collapsing every combination onto "/" stops the
   // homepage competing with itself for its own head terms.

nextjs-app/app/rankings/page.tsx

@@ -18,11 +18,8 @@ interface RankingsPageProps {
 }
 
 export const metadata: Metadata = {
-  // 'School Rankings' matched nothing anyone types. League tables is the
-  // phrase parents actually search, and it spikes each results day.
-  title: { absolute: 'Primary & Secondary School League Tables | schoolcompare' },
-  description:
-    'Rank English schools by SATs results, GCSEs, Progress 8 or Attainment 8, and filter by local authority or year. Built from the DfE’s own figures.',
+  title: 'School Rankings',
+  description: 'Top-ranked schools by SATs and GCSE performance across England',
   keywords: 'school rankings, top schools, best schools, KS2 rankings, KS4 rankings, school league tables',
   // Param forms (?metric=&local_authority=&year=&phase=) collapse here for
   // now. W3 replaces them with real indexable paths.

nextjs-app/components/school/AdmissionsSection.tsx

@@ -70,16 +70,10 @@ export function AdmissionsSection({
   {/* Spans both columns rather than taking a half-width cell. This is the
       figure parents come to the page for, and at tile width the two-line
       "0.31 miles / September 2025" pairing wraps badly. */}
-  {/* The {' '} between the figure and its metric support is not decoration.
-      The two are flex children, so the gap is drawn by CSS and the text layer
-      had nothing between them: textContent read "0.88 miles1.4 km", which is
-      what a screen reader announces and what any text matcher sees. Whitespace
-      text nodes are not rendered as flex items, so this changes the reading
-      without changing the layout. */}
   const distanceTile = cutoff && (
     <div className={`${styles.admissionsTile} ${styles.admissionsTileDistance}`}>
       <dd className={styles.admissionsTileNum}>
-        {cutoff.primary}{' '}
+        {cutoff.primary}
         <span className={styles.admissionsTileSub}>{cutoff.secondary}</span>
       </dd>
       <dt className={styles.admissionsTileLabel}>

nextjs-app/components/school/SecondaryAdmissionsSection.tsx

@@ -79,7 +79,7 @@ export function SecondaryAdmissionsSection({
               Last distance offered · {cutoff.entryYear}
             </div>
             <div className={styles.metricValue}>
-              {cutoff.primary}{' '}
+              {cutoff.primary}
               <span className={styles.metricValueSub}>{cutoff.secondary}</span>
             </div>
           </div>

nextjs-app/next.config.js

@@ -55,37 +55,6 @@ const nextConfig = {
   // Headers for caching and security
   async headers() {
     return [
-      {
-        /*
-         * Keep non-production hosts out of the index.
-         *
-         * Staging serves the same image as production off stx., so without
-         * this it is a full crawlable duplicate of the site.
-         *
-         * X-Robots-Tag, NOT a robots.txt Disallow. Disallow blocks crawling,
-         * which is not the same as blocking indexing — a disallowed URL can
-         * still be indexed from external links, and worse, blocking the crawl
-         * means Google never fetches the page and never sees a noindex at all.
-         * Staging therefore stays crawlable and answers "noindex" when crawled.
-         *
-         * Matched on the staging host explicitly rather than "any host that is
-         * not production". The inverted form is tempting because it would cover
-         * future environments automatically, but its failure mode is
-         * deindexing production if the Host header ever arrives rewritten by a
-         * proxy. This form's failure mode is a new environment being indexable
-         * until someone adds it here — recoverable, where the other is not.
-         *
-         * Any new non-production hostname must be added to this list.
-         */
-        source: '/:path*',
-        has: [{ type: 'host', value: 'stx.schoolcompare.co.uk' }],
-        headers: [
-          {
-            key: 'X-Robots-Tag',
-            value: 'noindex, nofollow',
-          },
-        ],
-      },
       {
         source: '/:path*',
         headers: [

pipeline/transform/dbt_project.yml

@@ -24,19 +24,6 @@ clean-targets: ["target", "dbt_packages"]
 vars:
   non_england_school_type_codes: [25, 26, 30, 37]
 
-  # Type alone is not enough. Beechwood College (URN 142458, Sully, CF64 5SE)
-  # is a Welsh school that GIAS types as a Special post 16 institution (32)
-  # rather than a Welsh establishment (30), so the type filter let it through.
-  # LA code is the reliable discriminator: GIAS gives the 22 Welsh unitary
-  # authorities the contiguous block 660-681, which English authorities never
-  # use (they occupy 201-384 and 800-938).
-  #
-  # Filtering on postcode would be wrong: Redbrook, Tutshill, Wyedean and two
-  # other Gloucestershire schools carry NP16/NP25 postcodes because Royal Mail
-  # areas straddle the border, and they are English schools with English data.
-  welsh_la_code_min: 660
-  welsh_la_code_max: 681
-
 models:
   school_compare:
     staging:

pipeline/transform/models/marts/dim_location.sql

@@ -37,5 +37,3 @@ where s.status_code in (1, 3)
 -- coalesce, not a bare NOT IN: a null type code would make the predicate
 -- null and drop the row silently. Unknown type is not grounds for exclusion.
 and coalesce(s.school_type_code, -1) not in ({{ var('non_england_school_type_codes') | join(', ') }})
--- Welsh schools GIAS does not type as Welsh establishments (see vars).
-and coalesce(s.local_authority_code, -1) not between {{ var('welsh_la_code_min') }} and {{ var('welsh_la_code_max') }}

pipeline/transform/models/marts/dim_school.sql

@@ -96,5 +96,3 @@ where s.status_code in (1, 3)
 -- coalesce, not a bare NOT IN: a null type code would make the predicate
 -- null and drop the row silently. Unknown type is not grounds for exclusion.
 and coalesce(s.school_type_code, -1) not in ({{ var('non_england_school_type_codes') | join(', ') }})
--- Welsh schools GIAS does not type as Welsh establishments (see vars).
-and coalesce(s.local_authority_code, -1) not between {{ var('welsh_la_code_min') }} and {{ var('welsh_la_code_max') }}

pipeline/transform/tests/assert_england_only_schools.sql

@@ -26,29 +26,3 @@ select
 from {{ ref('dim_location') }} l
 left join {{ ref('dim_school') }} s on l.urn = s.urn
 where s.urn is null
-
-union all
-
--- Belt and braces on the Welsh LA code range.
---
--- The range 660-681 is documented GIAS structure, not something this project
--- can verify from its own data — local_authority_code is not exposed anywhere
--- the app can assert on. So the range does the filtering and the authority
--- NAME does the checking: if 660-681 is ever wrong or incomplete, a Welsh
--- authority reappears here and the pipeline fails loudly rather than the
--- corpus quietly regrowing a Welsh tail.
---
--- Beechwood College (142458) is why this exists: a Welsh school GIAS types as
--- a Special post 16 institution, which the type filter alone let through.
-select
-    l.urn,
-    l.local_authority_name as school_name,
-    null::integer          as school_type_code
-from {{ ref('dim_location') }} l
-where l.local_authority_name in (
-    'Isle of Anglesey', 'Gwynedd', 'Conwy', 'Denbighshire', 'Flintshire',
-    'Wrexham', 'Powys', 'Ceredigion', 'Pembrokeshire', 'Carmarthenshire',
-    'Swansea', 'Neath Port Talbot', 'Bridgend', 'Vale of Glamorgan',
-    'Rhondda Cynon Taf', 'Merthyr Tydfil', 'Caerphilly', 'Blaenau Gwent',
-    'Torfaen', 'Monmouthshire', 'Newport', 'Cardiff'
-)