From 87642b7b061d0d057e29a9f2cb8d57ddc020c972 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 7 Jul 2026 09:22:54 +0100 Subject: [PATCH] fix(api): serialize schools that have no performance rows Schools without KS2/KS4 results (special post-16 institutions, sixth-form centres, PRUs, new schools) come back from the marts LEFT JOIN with NaN in every numeric column. school_info passed those raw pandas values straight into JSONResponse, which renders with allow_nan=False, so the detail endpoint 500d and the frontend turned that into a 404 on every such SEO landing page. Run school_info values through convert_to_native (the same treatment yearly_data already gets), add backend unit tests plus a pytest step in PR checks, and an e2e journey that finds a results-less school via the search API and asserts its page renders. Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/pr-checks.yml | 5 +- backend/app.py | 17 ++++- backend/tests/__init__.py | 0 .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 159 bytes ...chool_details.cpython-312-pytest-9.1.1.pyc | Bin 0 -> 5577 bytes backend/tests/test_school_details.py | 71 ++++++++++++++++++ e2e/tests/journeys.spec.ts | 28 +++++++ 7 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 backend/tests/__init__.py create mode 100644 backend/tests/__pycache__/__init__.cpython-312.pyc create mode 100644 backend/tests/__pycache__/test_school_details.cpython-312-pytest-9.1.1.pyc create mode 100644 backend/tests/test_school_details.py diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index be8783a..36a067b 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -51,11 +51,14 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install -r requirements.txt + run: pip install -r requirements.txt pytest "httpx<0.28" - name: Import smoke test run: python -c "from backend.app import app; print('backend imports OK')" + - name: Backend unit tests + run: python -m pytest backend/tests -q + build-backend: name: Build Backend (no push) runs-on: ubuntu-latest diff --git a/backend/app.py b/backend/app.py index 9a68e21..a146f19 100644 --- a/backend/app.py +++ b/backend/app.py @@ -33,7 +33,7 @@ from .data_loader import ( ) from .data_loader import get_data_info as get_db_info from .schemas import METRIC_DEFINITIONS, RANKING_COLUMNS, SCHOOL_COLUMNS -from .utils import clean_for_json +from .utils import clean_for_json, convert_to_native # Values to exclude from filter dropdowns (empty strings, non-applicable labels) EXCLUDED_FILTER_VALUES = {"", "Not applicable", "Does not apply"} @@ -582,8 +582,13 @@ async def get_school_details(request: Request, urn: int): except Exception: pass - return { - "school_info": { + # Schools with no performance rows (post-16 institutions, PRUs, new + # schools) carry NaN in every LEFT-JOINed numeric column; NaN reaching + # JSONResponse raises ValueError, so school_info needs the same + # conversion yearly_data gets from clean_for_json. + school_info = { + k: convert_to_native(v) + for k, v in { "urn": urn, "school_name": latest.get("school_name", ""), "local_authority": latest.get("local_authority", ""), @@ -601,7 +606,11 @@ async def get_school_details(request: Request, urn: int): "total_pupils": latest.get("gias_total_pupils"), "trust_name": latest.get("trust_name"), "gender": latest.get("gender"), - }, + }.items() + } + + return { + "school_info": school_info, "yearly_data": clean_for_json(school_data), # Supplementary data (null if not yet populated by Kestra) "ofsted": supplementary.get("ofsted"), diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/__pycache__/__init__.cpython-312.pyc b/backend/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea9f02d49a52b9e9150be8b0d7a38f3b7b3367a0 GIT binary patch literal 159 zcmX@j%ge<81dZE#vOx4>5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!veOSOPAw|dFDXsQ zFVZh4%FjwoE-BV8PR_{B&xudY&n-wSO4UzFOwLZtOVKY$Ee1-)$7kkcmc+;F6;$5h fu*uC&Da}c>D`Ev2#|Xs5AjU^#Mn=XWW*`dyVm~Ly literal 0 HcmV?d00001 diff --git a/backend/tests/__pycache__/test_school_details.cpython-312-pytest-9.1.1.pyc b/backend/tests/__pycache__/test_school_details.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c61f314afd38af1f9658e44f24439eb78bee1776 GIT binary patch literal 5577 zcmd^DU2Gf25#BoCyaHWwEyb#n<(YI7Qt5xYLuFxLP&3P~&;u9)8`7i2SfLz!!m5z!X$ zPJM;DUuI182CDFLNi%i>&9#_upsfWtRhPdlA#{agC}GTU9qbe4GrZtZJazFZ9yB~_ zkfpP(YqP<3^3+)!NF-)M882CW7CSEX2nGF|;iL(s?h?btnMcx=VPnr_e*f?|wjAbL ze%^-xSRZEAbwAtByCl-Y@nPgX{Q8wQfX-9HG5Z~|gv(>5aN5lgJa44m#tYQVVLwZ7 z&Y&>GOBW}v;#Xgro)Pm%%osB;3?|nIEnp{~Bh*SmXFH#B2JltrMI6&}Eyu^XJo7Qk z$F?y8EyK2!3AdZWe_s13HZWL9_*uiplo;tOVR+Uzs7-vI9Ao&^+1F;S5az)`V6le9 zp#4q5&XbFj!k!&BY};MZ9m6?0Y1oXU60jgKY3k;Eg2#r2V3|u#(~zE380!;541t>jb;<@&4N$*O2y2gTH$s|&}(_F zMys&2Ptg6()owEnfYo;MAkaf*2hhW2=U4ao@3opo-dBGvWu*J%-JM0LuH|39=8&?x za_{Gd=TXGQLLwwpG1+~5U^ z*e2Kp=eG+GSzg{Yd}8*6E-y?#B;{?N_1l*7HjEy6Klf*R+;Y-(-n5(yE)avJ27(B9 zkQMosi~=o;HE`M-dD`<9_!++BIwlPHdKlQe*Lfat=hKEsa)lu3WerAxx>=G2MS~Wq zHvED|mOEx^!XG~y{z1a05PxE@nV^jl+sasOp6MoW+??fzwJtYKxDZ#4>tn<7>_VVs zL}UbYMuzBs5gD=^9Uku=84HqznX?3PG2L@*D_scUX~Q$pmR|^x8Ovb0@A`(VdwI{Y zSCV>%O^p-yTD*W^$dko22Fh9ZQaQGS(kES>P0{bClkaB)8t^e zd~{^|6rORZpA|=g&tD@>p7fPBuMfX9hDYDH5X3nMrd^XPt6@#%y}y3m_2A>_moj(% zJ^c4y9)0+6N)6(0n1GNlaGmWsnUGcZSq1gaETBp3*t|`uE22v|om_`4=acu+eo6^s z&kX7&z=%FBW&Nrjv;q|vlNzS*RWbd7;rB=@GL&=W!FMKe0d$P2j%Wcs1;gx%D-hH!l zfXnSD*>h*2bdd9hQ1a0FXsLtqhf%WYL;0S1S1omN`3OpO-#Jz4;yebuV#kHj3tZ@i z0oGqB^>Dryb@o2)?0wRH=mYh(`UH3L{!e<2-W$I={-pQCdy{u3KOf|ZkwbNf?a1MJ zwS*2wReBV5DqTfv5Tfu3H2*O$aEIXc2b7amB?+ydMc%sWSph8ftB8D`_l4&xv=z8J zpdCHGF6@EOinLe8Fx6kg?w2zOpk922asl#E1OhZ@wq3)lo<6u;(3T;-&hikvBnK%A zAVTMumaa`ke8H{oZu5EHf}{k7r5W1G4akx11k2zav7wQZW1syVtb8OrmCv3{Ny{gW9`_0( zDVBJr%;2+_b3-XMz`)HR?efHwSjA_@r8wx zpa|6`Q}savLLKt|j=3Q5l2_nKLx<#*)FF%7kRd!;&MQGZH0$$u%l6^3AW*{q4%&5= zb@R3#wQV48LkyZYlRb`5RN%sE1Y6H2;tVopAegd#FEPyl3(oL!mrm{ z>U92T1gqm$cczaF4bj)Z#5s5wNBK4MSWCWl`rXsTwy$q$eQOhswWcktV?*mGD*fvW zNKxxpXB%4gmeLQuN9Br8s?rUm|A(WYw7J7R6v|=s18%OWe`n&}^xf&ISY^P@R{+lh zz|Zr6XOV-{uBQS9rhE`uI5e3N!O12H!6+X>kzEraJP+)80E9RcMW2PDnm+Me9Y9>! z3kdeos|5l%1OonfR1?Tr#d^(%djZ;W0vT3c_kX?mM0xevu0-^_)2|n&-|)YE`T*mN zGjtqWsrvn7IRG|EsDV3+Y%##_77p0MDTXa3kasl2;6?)2JIu-Ga$^lDkB6u{PCLN@ z+H+Ge`b}sIlttz`^gLI-z{w;h)115tq>P+_!lzdYkVEt`G}qRQ z)S~2At#Ih08B#;h!>q&$^ShZTcM;gx{vvLAdDb0?I9IctacB z)K2{B@`iRnz~;?_K+T?Yzo_(Z)bxv5&z8b}kIEIHv?-*Jf1rSV$ktL&0U*BFxS@3w zm1B2CffThaD79l-3jaMSSA^20kV5_e*ALlR3M$-U!-jUGs7w{Rr#J<^dul_Q+EV!M zQKcpnH-#AT4@8%cucZ%_DS_9ieL+muO*gIULA;V=Dc`){pdO~>fzV?hR}N6J1mCuh zVyjg_tXwCYA&A01X pd.DataFrame: + """One school row as produced by the marts query for a school with no + performance data: GIAS/location fields partly populated, every + results-linked column NaN (including year).""" + return pd.DataFrame( + [ + { + "urn": 150275, + "school_name": "West London Performing Arts Academy", + "phase": "Secondary", + "school_type": "Special post 16 institution", + "trust_name": None, + "religious_denomination": "Does not apply", + "gender": None, + "age_range": "16-25", + "admissions_policy": None, + "capacity": np.nan, + "gias_total_pupils": np.nan, + "headteacher_name": None, + "website": None, + "ofsted_grade": np.nan, + "local_authority": "Ealing", + "address": "268 Northfield Avenue, London, W5 4UB", + "postcode": "W5 4UB", + "latitude": 51.4986, + "longitude": -0.3148, + "year": np.nan, + "total_pupils": np.nan, + "eligible_pupils": np.nan, + "rwm_expected_pct": np.nan, + } + ] + ) + + +@pytest.fixture() +def client(monkeypatch): + from backend import app as app_module + + monkeypatch.setattr(app_module, "load_school_data", _no_results_school_df) + monkeypatch.setattr( + app_module, "get_supplementary_data", lambda db, urn: {} + ) + return TestClient(app_module.app, raise_server_exceptions=False) + + +def test_school_without_performance_rows_returns_200(client): + resp = client.get("/api/schools/150275") + assert resp.status_code == 200, resp.text + + +def test_nan_gias_fields_serialize_as_null(client): + info = client.get("/api/schools/150275").json()["school_info"] + assert info["capacity"] is None + assert info["total_pupils"] is None + assert info["school_name"] == "West London Performing Arts Academy" diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index dbb8923..cf68899 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -60,6 +60,34 @@ test('school detail page renders name and performance data', async ({ page }) => await expect(page.locator('canvas:visible').first()).toBeVisible({ timeout: 15_000 }); }); +test('school with no performance data still gets a working detail page', async ({ page }) => { + // Schools without KS2/KS4 results (special post-16 institutions, sixth-form + // centres, PRUs) used to 500 in the API — NaN GIAS fields broke JSON + // serialization — which the frontend rendered as a 404 on every such SEO + // landing page. Find one via the search API (year === null marks "no + // performance rows") and assert its page renders. + const candidates: number[] = []; + for (const q of ['post 16', 'specialist college', 'sixth form']) { + const resp = await page.request.get( + `/api/schools?search=${encodeURIComponent(q)}&per_page=20` + ); + if (!resp.ok()) continue; + const body = await resp.json(); + for (const s of body.schools ?? []) { + if (s.year === null && s.urn) candidates.push(s.urn); + } + if (candidates.length) break; + } + test.skip(candidates.length === 0, 'no results-less school in this dataset'); + + const detail = await page.request.get(`/api/schools/${candidates[0]}`); + expect(detail.status(), 'detail API must not 500 for a results-less school').toBe(200); + + await page.goto(`/school/${candidates[0]}`); + await page.waitForURL(/\/school\/\d+-/); // redirected to canonical slug + await expect(page.locator('h1').first()).toBeVisible(); +}); + test('school hero map opens fullscreen on mobile without the Fullscreen API', async ({ page }) => { // iOS Safari has no Element.requestFullscreen; the map must fall back to a // CSS overlay. Simulate that by removing the API before any page script runs.