diff --git a/backend/app.py b/backend/app.py index 45a7772..678fabc 100644 --- a/backend/app.py +++ b/backend/app.py @@ -72,41 +72,77 @@ def _school_url(urn: int, school_name: str) -> str: return f"/school/{urn}-{slug}" +# Routes worth submitting that are not a school page. /admissions was missing +# from the sitemap entirely despite being a static, indexable guide. +STATIC_SITEMAP_PATHS = ("/", "/rankings", "/compare", "/admissions") + + +def _has_publishable_data(row) -> bool: + """True when a school page has something a search result could state. + + A school with no results in any year and no Ofsted grade renders an empty + page. Submitting it spends crawl budget and drags the corpus-wide quality + signal down, so it stays out of the sitemap. The page itself still resolves + for anyone who has the URL. + """ + for field in ("rwm_expected_pct", "attainment_8_score", "ofsted_grade"): + value = row.get(field) + if value is not None and not pd.isna(value): + return True + return False + + +def _url_element(loc: str, lastmod: str | None = None) -> str: + """One entry. No priority or changefreq — Google ignores both.""" + body = f"{loc}" + if lastmod: + body += f"{lastmod}" + return f" {body}" + + +def _school_sitemap_rows(df) -> list[str]: + """A element per school that has something to show. + + lastmod comes from the school's Ofsted date where there is one and is + omitted otherwise. An always-now lastmod is a claim Google learns to + distrust; an absent one honestly means "unknown". + """ + if df.empty or "urn" not in df.columns or "school_name" not in df.columns: + return [] + + rows: list[str] = [] + seen: set[int] = set() + + # Latest row per URN first, so a school's most recent Ofsted date wins. + ordered = df.sort_values("year", ascending=False) if "year" in df.columns else df + + for _, row in ordered.iterrows(): + urn = int(row["urn"]) + if urn in seen: + continue + seen.add(urn) + if not _has_publishable_data(row): + continue + + lastmod = None + ofsted_date = row.get("ofsted_date") + if ofsted_date is not None and not pd.isna(ofsted_date): + lastmod = pd.Timestamp(ofsted_date).date().isoformat() + + rows.append(_url_element( + BASE_URL + _school_url(urn, str(row["school_name"])), lastmod)) + + return rows + + def build_sitemap() -> str: """Generate sitemap XML from in-memory school data. Returns the XML string.""" df = load_school_data() - static_urls = [ - (BASE_URL + "/", "daily", "1.0"), - (BASE_URL + "/rankings", "weekly", "0.8"), - (BASE_URL + "/compare", "weekly", "0.8"), - ] - lines = ['', ''] - - for url, freq, priority in static_urls: - lines.append( - f" {url}" - f"{freq}" - f"{priority}" - ) - - if not df.empty and "urn" in df.columns and "school_name" in df.columns: - seen = set() - for _, row in df[["urn", "school_name"]].drop_duplicates(subset="urn").iterrows(): - urn = int(row["urn"]) - name = str(row["school_name"]) - if urn in seen: - continue - seen.add(urn) - path = _school_url(urn, name) - lines.append( - f" {BASE_URL}{path}" - f"monthly" - f"0.6" - ) - + lines.extend(_url_element(BASE_URL + path) for path in STATIC_SITEMAP_PATHS) + lines.extend(_school_sitemap_rows(df)) lines.append("") return "\n".join(lines) diff --git a/backend/tests/test_sitemap.py b/backend/tests/test_sitemap.py index 430db84..7564cb1 100644 --- a/backend/tests/test_sitemap.py +++ b/backend/tests/test_sitemap.py @@ -42,3 +42,53 @@ def test_every_loc_uses_the_www_host(sitemap): # The apex 301s to www. A that redirects burns a crawl per URL. assert "https://www.schoolcompare.co.uk" in sitemap assert "https://schoolcompare.co.uk" not in sitemap + + +def test_school_with_results_is_listed(sitemap): + assert "/school/100001-alpha-primary" in sitemap + + +def test_school_with_no_results_and_no_ofsted_is_omitted(sitemap): + # Nothing for a search result to say about it. Submitting it spends crawl + # budget and drags the corpus-wide quality signal down. + assert "/school/100002" not in sitemap + + +def test_no_invented_priority_or_changefreq(sitemap): + # Google ignores both. They were noise dressed as signal. + assert "" not in sitemap + assert "" not in sitemap + + +def test_ofsted_date_becomes_lastmod(monkeypatch): + from backend import app as app_module + import datetime + + def _df(): + base = _schools_df() + base.loc[base["urn"] == 100001, "ofsted_date"] = datetime.date(2024, 3, 14) + return base + + monkeypatch.setattr(app_module, "load_school_data", _df) + xml = app_module.build_sitemap() + assert "2024-03-14" in xml + + +def test_no_lastmod_invented_when_date_unknown(monkeypatch): + # An always-now lastmod is a claim Google learns to distrust. Absent + # honestly means unknown. + from backend import app as app_module + + def _df(): + df = _schools_df() + df["ofsted_date"] = None + return df + + monkeypatch.setattr(app_module, "load_school_data", _df) + xml = app_module.build_sitemap() + assert "" not in xml + + +def test_static_routes_are_listed(sitemap): + for path in ("/", "/rankings", "/compare", "/admissions"): + assert f"https://www.schoolcompare.co.uk{path}" in sitemap