diff --git a/backend/app.py b/backend/app.py index 2b22d81..a3bfc4f 100644 --- a/backend/app.py +++ b/backend/app.py @@ -79,6 +79,12 @@ def _school_url(urn: int, school_name: str) -> str: STATIC_SITEMAP_PATHS = ("/", "/rankings", "/compare", "/admissions") +# A page has something a search result could state if any of these is present +# in any year. Shared by _has_publishable_data and the per-school check in +# _school_sitemap_rows so the two can never drift. +_PUBLISHABLE_FIELDS = ("rwm_expected_pct", "attainment_8_score", "ofsted_grade") + + def _has_publishable_data(row) -> bool: """True when a school page has something a search result could state. @@ -87,7 +93,7 @@ def _has_publishable_data(row) -> bool: signal down, so it stays out of the sitemap. The page itself still resolves for anyone who has the URL. """ - for field in ("rwm_expected_pct", "attainment_8_score", "ofsted_grade"): + for field in _PUBLISHABLE_FIELDS: value = row.get(field) if value is not None and not pd.isna(value): return True @@ -115,6 +121,21 @@ def _school_sitemap_rows(df) -> list[str]: rows: list[str] = [] seen: set[int] = set() + # Publishable is a property of the SCHOOL, not of its latest row. + # + # The first cut tested the latest year's row alone, which quietly dropped + # every school that has results in its history but a null row for the most + # recent year — a school that stopped reporting, or whose figures were + # suppressed for small-cohort disclosure. The Mallard Academy (150367) is + # the case that caught it: real KS2 results for 2015-16 through 2018-19, + # then null rows for 2022-23 onward. Its page shows all four years; the + # sitemap omitted it. Roughly 220 schools were affected. + publishable_cols = [c for c in _PUBLISHABLE_FIELDS if c in df.columns] + publishable: set[int] = ( + set(df.loc[df[publishable_cols].notna().any(axis=1), "urn"].astype(int)) + if publishable_cols else set() + ) + # Latest row per URN first, so a school's most recent Ofsted date wins. ordered = df.sort_values("year", ascending=False) if "year" in df.columns else df @@ -123,7 +144,7 @@ def _school_sitemap_rows(df) -> list[str]: if urn in seen: continue seen.add(urn) - if not _has_publishable_data(row): + if urn not in publishable: continue lastmod = None diff --git a/backend/tests/test_sitemap.py b/backend/tests/test_sitemap.py index eb030b1..c3d908a 100644 --- a/backend/tests/test_sitemap.py +++ b/backend/tests/test_sitemap.py @@ -176,3 +176,44 @@ def test_children_are_chunked_under_the_limit(monkeypatch): def test_build_sitemap_still_returns_the_index(sitemap): # lifespan and the admin endpoint call build_sitemap(); keep it working. assert "