feat(seo): rewrite the C1 snippets to earn the click (W8)

The baseline says these pages already rank and are not clicked. 'compare
school performance' sits at position 6.1 with 0.43% CTR; 'compare schools' at
7.2 with 0.87%. The brand query 'school compare' draws 9.16% from the same
neighbourhood of the same results page, which rules out a ranking explanation
— when the snippet gives a reason to click, it gets clicked.

These SERPs are owned by the DfE's own 'Compare school performance' service.
The old title put a lowercase brand nobody searches for in the most valuable
pixels, then a near-paraphrase of that service's name. Beside the government's
own result it read as a lookalike.

Intent in the title, differentiator in the description. Titles now match what
people type, and the descriptions carry the one fact gov.uk does not publish:
how close you had to live to get a place.

/compare deliberately takes the tool phrasing rather than the homepage's, so
the two pages stop competing for one phrase. The root layout's default and
Open Graph copy were saying something different again; they now agree.

No hard school counts in any of it. The corpus moves with every data refresh
and this repo has already shipped one copy bug of that kind.

Tests guard the mechanics — SERP length, intent keyword, the differentiator,
no brand-first title — and leave the wording free to iterate.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015mWQnpye9F299NVRCCSRvj

backend/app.py

@@ -79,6 +79,12 @@ def _school_url(urn: int, school_name: str) -> str:
 STATIC_SITEMAP_PATHS = ("/", "/rankings", "/compare", "/admissions")
 
 
+# A page has something a search result could state if any of these is present
+# in any year. Shared by _has_publishable_data and the per-school check in
+# _school_sitemap_rows so the two can never drift.
+_PUBLISHABLE_FIELDS = ("rwm_expected_pct", "attainment_8_score", "ofsted_grade")
+
+
 def _has_publishable_data(row) -> bool:
     """True when a school page has something a search result could state.
 
@@ -87,7 +93,7 @@ def _has_publishable_data(row) -> bool:
     signal down, so it stays out of the sitemap. The page itself still resolves
     for anyone who has the URL.
     """
-    for field in ("rwm_expected_pct", "attainment_8_score", "ofsted_grade"):
+    for field in _PUBLISHABLE_FIELDS:
         value = row.get(field)
         if value is not None and not pd.isna(value):
             return True
@@ -115,6 +121,21 @@ def _school_sitemap_rows(df) -> list[str]:
     rows: list[str] = []
     seen: set[int] = set()
 
+    # Publishable is a property of the SCHOOL, not of its latest row.
+    #
+    # The first cut tested the latest year's row alone, which quietly dropped
+    # every school that has results in its history but a null row for the most
+    # recent year — a school that stopped reporting, or whose figures were
+    # suppressed for small-cohort disclosure. The Mallard Academy (150367) is
+    # the case that caught it: real KS2 results for 2015-16 through 2018-19,
+    # then null rows for 2022-23 onward. Its page shows all four years; the
+    # sitemap omitted it. Roughly 220 schools were affected.
+    publishable_cols = [c for c in _PUBLISHABLE_FIELDS if c in df.columns]
+    publishable: set[int] = (
+        set(df.loc[df[publishable_cols].notna().any(axis=1), "urn"].astype(int))
+        if publishable_cols else set()
+    )
+
     # Latest row per URN first, so a school's most recent Ofsted date wins.
     ordered = df.sort_values("year", ascending=False) if "year" in df.columns else df
 
@@ -123,7 +144,7 @@ def _school_sitemap_rows(df) -> list[str]:
         if urn in seen:
             continue
         seen.add(urn)
-        if not _has_publishable_data(row):
+        if urn not in publishable:
             continue
 
         lastmod = None

backend/tests/test_sitemap.py

@@ -176,3 +176,44 @@ def test_children_are_chunked_under_the_limit(monkeypatch):
 def test_build_sitemap_still_returns_the_index(sitemap):
     # lifespan and the admin endpoint call build_sitemap(); keep it working.
     assert "<sitemapindex" in sitemap
+
+
+def test_school_with_results_in_an_earlier_year_is_still_listed(monkeypatch):
+    """Regression: The Mallard Academy (150367).
+
+    Real KS2 results 2015-16 to 2018-19, then null rows from 2022-23 onward
+    because the school stopped reporting. The first cut tested the latest
+    year's row alone and dropped it, along with ~220 others, even though its
+    detail page shows all four years of results.
+    """
+    from backend import app as app_module
+    import pandas as _pd
+
+    base = {"local_authority": "Testshire", "school_type": "Academy",
+            "phase": "Primary", "ofsted_date": None, "ofsted_grade": np.nan,
+            "attainment_8_score": np.nan, "urn": 150367,
+            "school_name": "Mallard Academy"}
+    df = _pd.DataFrame([
+        {**base, "year": 201819, "rwm_expected_pct": 67.0},
+        {**base, "year": 202324, "rwm_expected_pct": np.nan},
+        {**base, "year": 202425, "rwm_expected_pct": np.nan},
+    ])
+    monkeypatch.setattr(app_module, "load_school_data", lambda: df)
+
+    xml = app_module.build_sitemaps()["schools-1.xml"]
+    assert "/school/150367-mallard-academy" in xml
+
+
+def test_school_with_no_results_in_any_year_is_still_omitted(monkeypatch):
+    """The fix must not turn into "list everything"."""
+    from backend import app as app_module
+    import pandas as _pd
+
+    base = {"local_authority": "Testshire", "school_type": "Academy",
+            "phase": "Primary", "ofsted_date": None, "ofsted_grade": np.nan,
+            "attainment_8_score": np.nan, "rwm_expected_pct": np.nan,
+            "urn": 100002, "school_name": "Ghost Primary"}
+    df = _pd.DataFrame([{**base, "year": y} for y in (202324, 202425)])
+    monkeypatch.setattr(app_module, "load_school_data", lambda: df)
+
+    assert "/school/100002" not in app_module.build_sitemaps()["schools-1.xml"]

e2e/tests/journeys.spec.ts

@@ -1726,3 +1726,35 @@ test('a school page on staging is noindexed too, not just the homepage', async (
   const res = await page.request.get(`/school/${first.urn}-x`);
   expect(res.headers()['x-robots-tag']).toContain('noindex');
 });
+
+/*
+ * W8 — the C1 pages must ship a description, and it must differentiate.
+ *
+ * Baseline was 0.43% CTR at position 6.1 on "compare school performance",
+ * against 9.16% for the brand query from the same neighbourhood. The SERP is
+ * owned by the DfE's own service, so a description that paraphrases it earns
+ * nothing. Google may rewrite a snippet, but it cannot use one we never sent.
+ */
+test('every C1 page ships a description, and none opens its title with the brand', async ({ page }) => {
+  for (const path of ['/', '/compare', '/rankings', '/admissions']) {
+    await page.goto(path);
+
+    const desc = await page.locator('meta[name="description"]').first()
+      .getAttribute('content');
+    expect(desc, `${path} must ship a description`).toBeTruthy();
+    expect(desc!.length, `${path} description too short to be worth reading`)
+      .toBeGreaterThan(100);
+
+    const title = await page.title();
+    expect(title.toLowerCase().startsWith('schoolcompare'),
+      `${path} spends its most valuable pixels on the brand`).toBe(false);
+  }
+});
+
+test('the homepage snippet names what gov.uk does not publish', async ({ page }) => {
+  await page.goto('/');
+  const desc = await page.locator('meta[name="description"]').first()
+    .getAttribute('content');
+  // Admissions distance is the one fact the DfE service has no equivalent for.
+  expect(desc).toMatch(/close you had to live|distance/i);
+});

nextjs-app/__tests__/app/metadata.test.ts

@@ -51,3 +51,80 @@ describe('/compare indexability', () => {
       .toBe('https://www.schoolcompare.co.uk/compare');
   });
 });
+
+/*
+ * W8 — snippet copy for the C1 cluster.
+ *
+ * The baseline (GSC, 16 months to 2026-08-20) showed these pages ranking on
+ * page one and converting at a tenth of the normal rate: "compare school
+ * performance" at position 6.1 with 0.43% CTR, against 9.16% for the brand
+ * query from the same neighbourhood. The SERP is dominated by the DfE's own
+ * "Compare school performance" service, so the job of this copy is to say
+ * what that service does not offer, without losing intent match on the title.
+ *
+ * These tests guard the mechanics that make a snippet work — length, intent
+ * keyword, differentiator, no brand-first — not the exact wording, which
+ * should stay free to iterate.
+ */
+
+// Google truncates titles near 60 characters and descriptions near 155.
+const TITLE_MAX = 60;
+const DESC_MIN = 110;
+const DESC_MAX = 155;
+
+type Meta = { title?: unknown; description?: unknown };
+const titleOf = (m: Meta): string => {
+  const t = m.title as string | { absolute?: string } | undefined;
+  return typeof t === 'string' ? t : (t?.absolute ?? '');
+};
+
+describe('C1 snippet copy', () => {
+  const pages: Array<[string, Meta, RegExp]> = [
+    ['home', homeMetadata as Meta, /compare schools/i],
+    ['rankings', rankingsMetadata as Meta, /league table/i],
+    ['admissions', admissionsMetadata as Meta, /admission/i],
+  ];
+
+  for (const [name, meta, intent] of pages) {
+    it(`${name}: title carries the search intent and fits the SERP`, () => {
+      const t = titleOf(meta);
+      expect(t).toMatch(intent);
+      expect(t.length).toBeLessThanOrEqual(TITLE_MAX);
+    });
+
+    it(`${name}: title does not open with the brand`, () => {
+      // The measured 0.43% CTR came from a brand-first title. The most
+      // valuable pixels go to the thing the searcher typed.
+      expect(titleOf(meta).toLowerCase().startsWith('schoolcompare')).toBe(false);
+    });
+
+    it(`${name}: description is long enough to be worth reading, short enough to survive`, () => {
+      const d = meta.description as string;
+      expect(d.length).toBeGreaterThanOrEqual(DESC_MIN);
+      expect(d.length).toBeLessThanOrEqual(DESC_MAX);
+    });
+  }
+
+  it('the homepage description names what gov.uk does not publish', () => {
+    // Admissions distance is the one fact the DfE service has no equivalent
+    // for. If it ever leaves this description, the snippet is competing with
+    // gov.uk on gov.uk's own ground.
+    expect(homeMetadata.description).toMatch(/close you had to live|distance/i);
+  });
+
+  it('/compare targets the tool phrasing rather than repeating the homepage', () => {
+    // Two pages chasing one phrase is how a site competes with itself.
+    return compareMetadata({ searchParams: Promise.resolve({}) }).then((m) => {
+      expect(m.title).toMatch(/comparison tool/i);
+      expect(m.title).not.toBe(titleOf(homeMetadata as Meta));
+    });
+  });
+
+  it('no C1 page claims a school count that will drift', () => {
+    // The corpus moves with every data refresh; this repo has already shipped
+    // one copy bug of that kind ("three schools" against MAX_SCHOOLS = 5).
+    for (const [, meta] of pages) {
+      expect(meta.description as string).not.toMatch(/\b\d{2},\d{3}\b|\b\d{2},000\b/);
+    }
+  });
+});

nextjs-app/app/admissions/page.tsx

@@ -5,9 +5,11 @@ import { AdmissionsView } from '@/components/AdmissionsView';
 export const dynamic = 'force-static';
 
 export const metadata: Metadata = {
-  title: 'School Admissions Guide',
+  // Deadlines and offer days are what gets searched, and what this page is
+  // genuinely best at — the countdowns are live.
+  title: { absolute: 'School Admissions Deadlines & Offer Days | schoolcompare' },
   description:
-    'Understand the Primary and Secondary school admissions process in England, with live countdowns to every key deadline and National Offer Day.',
+    'Every key date for primary and secondary school admissions in England, with live countdowns to the application deadline and National Offer Day.',
   alternates: { canonical: absoluteUrl('/admissions') },
 };
 

nextjs-app/app/compare/page.tsx

@@ -30,9 +30,12 @@ export async function generateMetadata(
   const { urns } = await searchParams;
 
   const base: Metadata = {
-    title: 'Compare Schools',
+    // Deliberately not the homepage's phrase. Two pages chasing "compare
+    // schools" is how a site competes with itself; this one takes the tool
+    // phrasing instead.
+    title: 'School Comparison Tool — Up to Five at Once | schoolcompare',
     description:
-      'Compare schools in England side by side — Ofsted inspections, KS2 and GCSE results against the England average, admissions odds and school community.',
+      'Put up to five English schools in one table: SATs and GCSE results against the England average, Ofsted grades, and the distance places were offered.',
     keywords:
       'school comparison, compare schools, Ofsted comparison, school admissions, KS2 comparison, primary school performance',
     alternates: { canonical: absoluteUrl('/compare') },

nextjs-app/app/layout.tsx

@@ -48,10 +48,11 @@ export const metadata: Metadata = {
     statusBarStyle: 'default',
   },
   title: {
-    default: 'schoolcompare | Compare School Performance',
+    default: 'Compare Schools Side by Side | schoolcompare',
     template: '%s | schoolcompare',
   },
-  description: 'Compare primary and secondary school SATs and GCSE performance across England',
+  description:
+    'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place. Free, no sign-up.',
   keywords: 'school comparison, KS2 results, KS4 results, primary school, secondary school, England schools, SATs results, GCSE results',
   authors: [{ name: 'schoolcompare' }],
   manifest: '/manifest.json',
@@ -61,16 +62,18 @@ export const metadata: Metadata = {
   metadataBase: new URL(SITE_URL),
   openGraph: {
     type: 'website',
-    title: 'schoolcompare | Compare School Performance',
-    description: 'Compare primary and secondary school SATs and GCSE performance across England',
+    title: 'Compare Schools Side by Side | schoolcompare',
+    description:
+      'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place.',
     url: SITE_URL,
     siteName: 'schoolcompare',
   },
   twitter: {
     // summary_large_image now that there is an image worth showing.
     card: 'summary_large_image',
-    title: 'schoolcompare | Compare School Performance',
-    description: 'Compare primary and secondary school SATs and GCSE performance across England',
+    title: 'Compare Schools Side by Side | schoolcompare',
+    description:
+      'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place.',
   },
 };
 

nextjs-app/app/page.tsx

@@ -34,8 +34,21 @@ interface HomePageProps {
  * saying the brand twice.
  */
 export const metadata: Metadata = {
-  title: { absolute: 'schoolcompare | Compare every school in England' },
-  description: 'Search and compare school performance across England',
+  /*
+   * Intent in the title, differentiator in the description.
+   *
+   * These queries are owned by the DfE's own "Compare school performance"
+   * service, and the old title — brand first, then a near-paraphrase of that
+   * service's name — gave a searcher no reason to pick us over it. It drew
+   * 0.43% CTR at position 6.1 while the brand query drew 9.16% from the same
+   * neighbourhood, so the ranking was never the problem.
+   *
+   * The title now matches what people type. The description carries the one
+   * fact gov.uk does not publish: how close you had to live to get a place.
+   */
+  title: { absolute: 'Compare Schools Side by Side | schoolcompare' },
+  description:
+    'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place. Free, no sign-up.',
   // This page reads eleven search params. They filter a result set; they do
   // not make a new document. Collapsing every combination onto "/" stops the
   // homepage competing with itself for its own head terms.

nextjs-app/app/rankings/page.tsx

@@ -18,8 +18,11 @@ interface RankingsPageProps {
 }
 
 export const metadata: Metadata = {
-  title: 'School Rankings',
-  description: 'Top-ranked schools by SATs and GCSE performance across England',
+  // 'School Rankings' matched nothing anyone types. League tables is the
+  // phrase parents actually search, and it spikes each results day.
+  title: { absolute: 'Primary & Secondary School League Tables | schoolcompare' },
+  description:
+    'Rank English schools by SATs results, GCSEs, Progress 8 or Attainment 8, and filter by local authority or year. Built from the DfE’s own figures.',
   keywords: 'school rankings, top schools, best schools, KS2 rankings, KS4 rankings, school league tables',
   // Param forms (?metric=&local_authority=&year=&phase=) collapse here for
   // now. W3 replaces them with real indexable paths.