From 07c97a46c500d6285ac55884b20da602810e7444 Mon Sep 17 00:00:00 2001 From: Tudor Date: Fri, 21 Aug 2026 00:04:32 +0100 Subject: [PATCH] fix(seo): a school is publishable on any year's results, not the latest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _school_sitemap_rows tested only the latest year's row, which quietly dropped every school with results in its history but a null row for the most recent year — a school that stopped reporting, or whose figures were suppressed for small-cohort disclosure. The Mallard Academy (150367) is the case that caught it: real KS2 results for 2015-16 through 2018-19, then null rows from 2022-23 on. Its detail page shows all four years; the sitemap omitted it. Sampling 40 of the 2,206 excluded schools found 4 like this, so roughly 220 real pages were being withheld. Publishable is now a property of the school, computed across every row, while lastmod still comes from the latest row so the most recent Ofsted date wins. The field list is a module constant shared with _has_publishable_data so the two checks cannot drift. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015mWQnpye9F299NVRCCSRvj --- backend/app.py | 25 +++++++++++++++++++-- backend/tests/test_sitemap.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/backend/app.py b/backend/app.py index 2b22d81..a3bfc4f 100644 --- a/backend/app.py +++ b/backend/app.py @@ -79,6 +79,12 @@ def _school_url(urn: int, school_name: str) -> str: STATIC_SITEMAP_PATHS = ("/", "/rankings", "/compare", "/admissions") +# A page has something a search result could state if any of these is present +# in any year. Shared by _has_publishable_data and the per-school check in +# _school_sitemap_rows so the two can never drift. +_PUBLISHABLE_FIELDS = ("rwm_expected_pct", "attainment_8_score", "ofsted_grade") + + def _has_publishable_data(row) -> bool: """True when a school page has something a search result could state. @@ -87,7 +93,7 @@ def _has_publishable_data(row) -> bool: signal down, so it stays out of the sitemap. The page itself still resolves for anyone who has the URL. """ - for field in ("rwm_expected_pct", "attainment_8_score", "ofsted_grade"): + for field in _PUBLISHABLE_FIELDS: value = row.get(field) if value is not None and not pd.isna(value): return True @@ -115,6 +121,21 @@ def _school_sitemap_rows(df) -> list[str]: rows: list[str] = [] seen: set[int] = set() + # Publishable is a property of the SCHOOL, not of its latest row. + # + # The first cut tested the latest year's row alone, which quietly dropped + # every school that has results in its history but a null row for the most + # recent year — a school that stopped reporting, or whose figures were + # suppressed for small-cohort disclosure. The Mallard Academy (150367) is + # the case that caught it: real KS2 results for 2015-16 through 2018-19, + # then null rows for 2022-23 onward. Its page shows all four years; the + # sitemap omitted it. Roughly 220 schools were affected. + publishable_cols = [c for c in _PUBLISHABLE_FIELDS if c in df.columns] + publishable: set[int] = ( + set(df.loc[df[publishable_cols].notna().any(axis=1), "urn"].astype(int)) + if publishable_cols else set() + ) + # Latest row per URN first, so a school's most recent Ofsted date wins. ordered = df.sort_values("year", ascending=False) if "year" in df.columns else df @@ -123,7 +144,7 @@ def _school_sitemap_rows(df) -> list[str]: if urn in seen: continue seen.add(urn) - if not _has_publishable_data(row): + if urn not in publishable: continue lastmod = None diff --git a/backend/tests/test_sitemap.py b/backend/tests/test_sitemap.py index eb030b1..c3d908a 100644 --- a/backend/tests/test_sitemap.py +++ b/backend/tests/test_sitemap.py @@ -176,3 +176,44 @@ def test_children_are_chunked_under_the_limit(monkeypatch): def test_build_sitemap_still_returns_the_index(sitemap): # lifespan and the admin endpoint call build_sitemap(); keep it working. assert "