From a524cdc591fd2af7611c29dadcd5e1e05e180676 Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 7 Jul 2026 13:33:25 +0100 Subject: [PATCH] fix(api): survive missing has_sixth_form column and numpy bool serialization - data_loader.load_school_data_as_dataframe now catches a ProgrammingError whose message mentions has_sixth_form (psycopg2 UndefinedColumn) and retries with a NULL-AS-has_sixth_form query variant, so the API keeps serving data (and the app.py column-fallback branch stays reachable) even before the nightly pipeline has rebuilt marts.dim_school. - utils.convert_to_native now handles numpy.bool_ so GET /api/schools/{urn} doesn't 500 once has_sixth_form is a populated bool-dtype column. - Update the now-stale comment on the app.py age-range fallback branch. Co-Authored-By: Claude Fable 5 --- backend/app.py | 5 +- backend/data_loader.py | 28 +++++++++ backend/tests/test_sixth_form_flag.py | 81 +++++++++++++++++++++++++++ backend/utils.py | 2 + 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/backend/app.py b/backend/app.py index 9b63008..7f017df 100644 --- a/backend/app.py +++ b/backend/app.py @@ -421,7 +421,10 @@ async def get_schools( if has_sixth_form in ("yes", "no"): if "has_sixth_form" in df_latest.columns: flag = df_latest["has_sixth_form"].eq(True) - else: # DB predates the pipeline re-run — fall back to age range + else: # Defensive fallback only — data_loader now always synthesizes + # has_sixth_form as NULL when the DB predates the pipeline re-run, + # so this branch shouldn't normally trigger. Falls back to age + # range if the column is somehow absent anyway. flag = df_latest["age_range"].str.contains("18", na=False) df_latest = df_latest[flag if has_sixth_form == "yes" else ~flag] diff --git a/backend/data_loader.py b/backend/data_loader.py index 6221461..9e50c73 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -3,11 +3,14 @@ Data loading module — reads from marts.* tables built by dbt. Provides efficient queries with caching. """ +import logging + import pandas as pd import numpy as np from typing import Optional, Dict, Tuple, List import requests from sqlalchemy import text +import sqlalchemy.exc from sqlalchemy.orm import Session from .config import settings @@ -215,11 +218,36 @@ _MAIN_QUERY = text(""" ORDER BY s.school_name, p.year """) +# Fallback used when marts.dim_school predates the has_sixth_form column +# (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on this DB). +# Keeps the column present as NULL so downstream code — including the +# app.py fallback branch — behaves as designed instead of KeyError-ing. +_MAIN_QUERY_NO_SIXTH_FORM = text( + str(_MAIN_QUERY).replace("s.has_sixth_form,", "NULL AS has_sixth_form,") +) +assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), ( + "expected replacement of 's.has_sixth_form,' to have taken effect" +) + def load_school_data_as_dataframe() -> pd.DataFrame: """Load all school + KS2 data as a pandas DataFrame.""" try: df = pd.read_sql(_MAIN_QUERY, engine) + except sqlalchemy.exc.ProgrammingError as exc: + if "has_sixth_form" not in str(exc): + print(f"Warning: Could not load school data from marts: {exc}") + return pd.DataFrame() + logging.getLogger(__name__).warning( + "marts.dim_school is missing has_sixth_form (pipeline hasn't " + "rebuilt the mart yet on this DB) — retrying without it: %s", + exc, + ) + try: + df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine) + except Exception as exc2: + print(f"Warning: Could not load school data from marts: {exc2}") + return pd.DataFrame() except Exception as exc: print(f"Warning: Could not load school data from marts: {exc}") return pd.DataFrame() diff --git a/backend/tests/test_sixth_form_flag.py b/backend/tests/test_sixth_form_flag.py index fe1782a..fb7c34c 100644 --- a/backend/tests/test_sixth_form_flag.py +++ b/backend/tests/test_sixth_form_flag.py @@ -95,3 +95,84 @@ def test_detail_payload_includes_flag(client): resp = client.get("/api/schools/100002") assert resp.status_code == 200, resp.text assert resp.json()["school_info"]["has_sixth_form"] is True + + +def test_detail_payload_serializes_numpy_bool(monkeypatch): + """Once the pipeline has run, has_sixth_form is a real bool dtype column + (dbt not_null test guarantees no NULLs), so row access yields + numpy.bool_ rather than a Python bool. convert_to_native must handle it — + otherwise FastAPI's jsonable_encoder raises ValueError and the detail + endpoint 500s (C2).""" + from backend import app as app_module + + df = _schools_df() + # Drop the row with a None flag — this fixture models the post-pipeline + # state where the column is a genuine, fully-populated bool dtype. + df = df[df["has_sixth_form"].notna()].reset_index(drop=True) + df["has_sixth_form"] = df["has_sixth_form"].astype(bool) + assert df["has_sixth_form"].dtype == bool + + monkeypatch.setattr(app_module, "load_school_data", lambda: df) + monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {}) + client = TestClient(app_module.app, raise_server_exceptions=False) + + resp = client.get("/api/schools/100002") + assert resp.status_code == 200, resp.text + assert resp.json()["school_info"]["has_sixth_form"] is True + + +class _FakeProgrammingError(Exception): + """Stand-in for sqlalchemy.exc.ProgrammingError wrapping psycopg2's + UndefinedColumn, without needing a real DB connection to construct one.""" + + +def test_load_school_data_survives_missing_has_sixth_form_column(monkeypatch): + """Real prod state until the nightly pipeline first rebuilds the mart: + marts.dim_school lacks has_sixth_form entirely. The first query raises + UndefinedColumn; load_school_data_as_dataframe must retry without the + column (synthesizing it as None) rather than swallow the error and + return (and then have load_school_data cache) an empty DataFrame (C1).""" + import sqlalchemy.exc + from backend import data_loader + + data_loader._df_cache = None + data_loader._df_latest_cache = None + + good_df = pd.DataFrame( + [ + { + "urn": 1, + "school_name": "Fallback School", + "school_type": "Academy", + "has_sixth_form": None, + } + ] + ) + calls = [] + + def fake_read_sql(query, con): + calls.append(query) + if len(calls) == 1: + raise sqlalchemy.exc.ProgrammingError( + "SELECT ...", + None, + Exception( + "(psycopg2.errors.UndefinedColumn) column s.has_sixth_form " + "does not exist" + ), + ) + return good_df.copy() + + monkeypatch.setattr(data_loader.pd, "read_sql", fake_read_sql) + + try: + df = data_loader.load_school_data_as_dataframe() + finally: + data_loader._df_cache = None + data_loader._df_latest_cache = None + + assert len(calls) == 2, "must retry with the no-sixth-form query variant" + assert calls[1] is data_loader._MAIN_QUERY_NO_SIXTH_FORM + assert not df.empty + assert "has_sixth_form" in df.columns + assert df["has_sixth_form"].iloc[0] is None diff --git a/backend/utils.py b/backend/utils.py index 18fb37e..01f885c 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -11,6 +11,8 @@ def convert_to_native(value: Any) -> Any: """Convert numpy types to native Python types for JSON serialization.""" if pd.isna(value): return None + if isinstance(value, np.bool_): + return bool(value) if isinstance(value, (np.integer,)): return int(value) if isinstance(value, (np.floating,)):