school_compare/backend/tests/test_sitemap.py

"""Tests for sitemap generation (spec 2026-08-20, workstream W1).

The sitemap is built from the in-memory school DataFrame, so these inject a
small frame via monkeypatch rather than touching a database.
"""

import numpy as np
import pandas as pd
import pytest


def _schools_df() -> pd.DataFrame:
    """Two schools: one with results, one with neither results nor Ofsted."""
    base = {
        "local_authority": "Testshire",
        "school_type": "Academy",
        "phase": "Primary",
        "year": 202425,
        "ofsted_date": None,
    }
    return pd.DataFrame(
        [
            {**base, "urn": 100001, "school_name": "Alpha Primary",
             "rwm_expected_pct": 62.0, "attainment_8_score": np.nan,
             "ofsted_grade": 2.0},
            {**base, "urn": 100002, "school_name": "Ghost Primary",
             "rwm_expected_pct": np.nan, "attainment_8_score": np.nan,
             "ofsted_grade": np.nan},
        ]
    )


@pytest.fixture()
def sitemap(monkeypatch) -> str:
    """The sitemap index."""
    from backend import app as app_module

    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
    return app_module.build_sitemap()


@pytest.fixture()
def schools_child(monkeypatch) -> str:
    """The first school child sitemap, where school URLs actually live."""
    from backend import app as app_module

    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
    return app_module.build_sitemaps()["schools-1.xml"]


@pytest.fixture()
def static_child(monkeypatch) -> str:
    from backend import app as app_module

    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
    return app_module.build_sitemaps()["static.xml"]


def test_every_loc_uses_the_www_host(sitemaps):
    # The apex 301s to www. A <loc> that redirects burns a crawl per URL.
    # Checked across every file, index included, not just one.
    #
    # A child can legitimately be empty — this fixture holds two schools and no
    # town clearing the threshold — so the presence check applies only to files
    # that carry URLs. The absence check applies to all of them.
    for name, xml in sitemaps.items():
        assert "https://schoolcompare.co.uk" not in xml, name
        if "<loc>" in xml:
            assert "https://www.schoolcompare.co.uk" in xml, name


def test_school_with_results_is_listed(schools_child):
    assert "/school/100001-alpha-primary" in schools_child


def test_school_with_no_results_and_no_ofsted_is_omitted(schools_child):
    # Nothing for a search result to say about it. Submitting it spends crawl
    # budget and drags the corpus-wide quality signal down.
    #
    # Asserted against the child, not the index: the index carries no school
    # URLs at all, so it would pass this trivially and prove nothing.
    assert "/school/100002" not in schools_child


def test_no_invented_priority_or_changefreq(sitemaps):
    # Google ignores both. They were noise dressed as signal.
    for name, xml in sitemaps.items():
        assert "<priority>" not in xml, name
        assert "<changefreq>" not in xml, name


def test_ofsted_date_becomes_lastmod(monkeypatch):
    from backend import app as app_module
    import datetime

    def _df():
        base = _schools_df()
        base.loc[base["urn"] == 100001, "ofsted_date"] = datetime.date(2024, 3, 14)
        return base

    monkeypatch.setattr(app_module, "load_school_data", _df)
    xml = app_module.build_sitemaps()["schools-1.xml"]
    assert "<lastmod>2024-03-14</lastmod>" in xml


def test_no_lastmod_invented_when_date_unknown(monkeypatch):
    # An always-now lastmod is a claim Google learns to distrust. Absent
    # honestly means unknown.
    from backend import app as app_module

    def _df():
        df = _schools_df()
        df["ofsted_date"] = None
        return df

    monkeypatch.setattr(app_module, "load_school_data", _df)
    # The child only. The index legitimately carries a lastmod, because there
    # it means "when this sitemap file changed", which we do know.
    xml = app_module.build_sitemaps()["schools-1.xml"]
    assert "<lastmod>" not in xml


def test_static_routes_are_listed(static_child):
    for path in ("/", "/rankings", "/compare", "/admissions"):
        assert f"<loc>https://www.schoolcompare.co.uk{path}</loc>" in static_child


@pytest.fixture()
def sitemaps(monkeypatch) -> dict:
    from backend import app as app_module

    monkeypatch.setattr(app_module, "load_school_data", _schools_df)
    return app_module.build_sitemaps()


def test_index_lists_each_child(sitemaps):
    index = sitemaps["sitemap.xml"]
    assert "<sitemapindex" in index
    assert "https://www.schoolcompare.co.uk/sitemaps/static.xml" in index
    assert "https://www.schoolcompare.co.uk/sitemaps/schools-1.xml" in index


def test_index_carries_no_url_elements(sitemaps):
    # A sitemap index holds <sitemap> entries only; mixing in <url> is invalid.
    assert "<url>" not in sitemaps["sitemap.xml"]


def test_index_does_not_list_itself(sitemaps):
    assert "<loc>https://www.schoolcompare.co.uk/sitemap.xml</loc>" not in sitemaps["sitemap.xml"]


def test_static_child_holds_the_static_routes(sitemaps):
    static = sitemaps["static.xml"]
    for path in ("/", "/rankings", "/compare", "/admissions"):
        assert f"<loc>https://www.schoolcompare.co.uk{path}</loc>" in static


def test_school_child_holds_the_schools(sitemaps):
    assert "/school/100001-alpha-primary" in sitemaps["schools-1.xml"]


def test_children_are_chunked_under_the_limit(monkeypatch):
    # Sitemaps cap at 50,000 URLs per file. Chunk at 10,000 so a child stays
    # small enough to eyeball in Search Console.
    from backend import app as app_module
    import pandas as _pd

    rows = [
        {"urn": 200000 + i, "school_name": f"School {i}", "year": 202425,
         "rwm_expected_pct": 60.0, "attainment_8_score": None,
         "ofsted_grade": 2.0, "ofsted_date": None}
        for i in range(10_001)
    ]
    monkeypatch.setattr(app_module, "load_school_data", lambda: _pd.DataFrame(rows))

    maps = app_module.build_sitemaps()
    assert maps["schools-1.xml"].count("<url>") == 10_000
    assert maps["schools-2.xml"].count("<url>") == 1


def test_build_sitemap_still_returns_the_index(sitemap):
    # lifespan and the admin endpoint call build_sitemap(); keep it working.
    assert "<sitemapindex" in sitemap


def test_school_with_results_in_an_earlier_year_is_still_listed(monkeypatch):
    """Regression: The Mallard Academy (150367).

    Real KS2 results 2015-16 to 2018-19, then null rows from 2022-23 onward
    because the school stopped reporting. The first cut tested the latest
    year's row alone and dropped it, along with ~220 others, even though its
    detail page shows all four years of results.
    """
    from backend import app as app_module
    import pandas as _pd

    base = {"local_authority": "Testshire", "school_type": "Academy",
            "phase": "Primary", "ofsted_date": None, "ofsted_grade": np.nan,
            "attainment_8_score": np.nan, "urn": 150367,
            "school_name": "Mallard Academy"}
    df = _pd.DataFrame([
        {**base, "year": 201819, "rwm_expected_pct": 67.0},
        {**base, "year": 202324, "rwm_expected_pct": np.nan},
        {**base, "year": 202425, "rwm_expected_pct": np.nan},
    ])
    monkeypatch.setattr(app_module, "load_school_data", lambda: df)

    xml = app_module.build_sitemaps()["schools-1.xml"]
    assert "/school/150367-mallard-academy" in xml


def test_school_with_no_results_in_any_year_is_still_omitted(monkeypatch):
    """The fix must not turn into "list everything"."""
    from backend import app as app_module
    import pandas as _pd

    base = {"local_authority": "Testshire", "school_type": "Academy",
            "phase": "Primary", "ofsted_date": None, "ofsted_grade": np.nan,
            "attainment_8_score": np.nan, "rwm_expected_pct": np.nan,
            "urn": 100002, "school_name": "Ghost Primary"}
    df = _pd.DataFrame([{**base, "year": y} for y in (202324, 202425)])
    monkeypatch.setattr(app_module, "load_school_data", lambda: df)

    assert "/school/100002" not in app_module.build_sitemaps()["schools-1.xml"]


def _places_df() -> pd.DataFrame:
    base = {
        "local_authority": "Essex", "school_type": "Academy",
        "phase": "Primary", "year": 202425, "ofsted_grade": 2.0,
        "ofsted_date": None, "attainment_8_score": np.nan,
        "town": "Brentwood", "postcode": "CM13 1AA",
    }
    return pd.DataFrame([
        {**base, "urn": 100000 + i, "school_name": f"Brentwood School {i}",
         "rwm_expected_pct": 60.0}
        for i in range(6)
    ])


@pytest.fixture()
def place_sitemaps(monkeypatch) -> dict:
    from backend import app as app_module

    monkeypatch.setattr(app_module, "load_school_data", _places_df)
    monkeypatch.setattr(app_module, "_place_registry", None)
    return app_module.build_sitemaps()


def test_place_children_are_listed_in_the_index(place_sitemaps):
    index = place_sitemaps["sitemap.xml"]
    assert "/sitemaps/places-1.xml" in index
    assert "/sitemaps/outcodes-1.xml" in index


def test_town_and_authority_urls_use_their_own_namespaces(place_sitemaps):
    xml = place_sitemaps["places-1.xml"]
    assert "<loc>https://www.schoolcompare.co.uk/schools/brentwood</loc>" in xml
    assert "<loc>https://www.schoolcompare.co.uk/schools/authority/essex</loc>" in xml


def test_outcode_urls_live_in_their_own_child(place_sitemaps):
    assert "/schools/near/cm13" in place_sitemaps["outcodes-1.xml"]
    assert "/schools/near/cm13" not in place_sitemaps["places-1.xml"]


def test_place_urls_carry_no_priority_or_changefreq(place_sitemaps):
    for name in ("places-1.xml", "outcodes-1.xml"):
        assert "<priority>" not in place_sitemaps[name]
        assert "<changefreq>" not in place_sitemaps[name]


def test_phase_variants_are_submitted_where_the_phase_clears_the_threshold(place_sitemaps):
    # "primary schools in beccles" is the query shape the baseline showed, so
    # each variant is its own page and has to be submitted. Emitting only the
    # bare place URL left ~950 of them reachable by nothing.
    xml = place_sitemaps["places-1.xml"]
    assert "<loc>https://www.schoolcompare.co.uk/schools/brentwood/primary</loc>" in xml


def test_a_phase_below_its_own_threshold_is_not_submitted(place_sitemaps):
    # The fixture is six primaries and no secondaries.
    xml = place_sitemaps["places-1.xml"]
    assert "/schools/brentwood/secondary" not in xml


def test_outcodes_get_no_phase_variants(place_sitemaps):
    # Nobody searches "primary schools in CM13"; the routes do not exist.
    xml = place_sitemaps["outcodes-1.xml"]
    assert "/primary" not in xml and "/secondary" not in xml