fix(api): survive missing has_sixth_form column and numpy bool serialization

- data_loader.load_school_data_as_dataframe now catches a ProgrammingError
  whose message mentions has_sixth_form (psycopg2 UndefinedColumn) and
  retries with a NULL-AS-has_sixth_form query variant, so the API keeps
  serving data (and the app.py column-fallback branch stays reachable)
  even before the nightly pipeline has rebuilt marts.dim_school.
- utils.convert_to_native now handles numpy.bool_ so GET /api/schools/{urn}
  doesn't 500 once has_sixth_form is a populated bool-dtype column.
- Update the now-stale comment on the app.py age-range fallback branch.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/app.py

@@ -421,7 +421,10 @@ async def get_schools(
     if has_sixth_form in ("yes", "no"):
         if "has_sixth_form" in df_latest.columns:
             flag = df_latest["has_sixth_form"].eq(True)
-        else:  # DB predates the pipeline re-run — fall back to age range
+        else:  # Defensive fallback only — data_loader now always synthesizes
+            # has_sixth_form as NULL when the DB predates the pipeline re-run,
+            # so this branch shouldn't normally trigger. Falls back to age
+            # range if the column is somehow absent anyway.
             flag = df_latest["age_range"].str.contains("18", na=False)
         df_latest = df_latest[flag if has_sixth_form == "yes" else ~flag]
 

backend/data_loader.py

@@ -3,11 +3,14 @@ Data loading module — reads from marts.* tables built by dbt.
 Provides efficient queries with caching.
 """
 
+import logging
+
 import pandas as pd
 import numpy as np
 from typing import Optional, Dict, Tuple, List
 import requests
 from sqlalchemy import text
+import sqlalchemy.exc
 from sqlalchemy.orm import Session
 
 from .config import settings
@@ -215,11 +218,36 @@ _MAIN_QUERY = text("""
     ORDER BY s.school_name, p.year
 """)
 
+# Fallback used when marts.dim_school predates the has_sixth_form column
+# (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on this DB).
+# Keeps the column present as NULL so downstream code — including the
+# app.py fallback branch — behaves as designed instead of KeyError-ing.
+_MAIN_QUERY_NO_SIXTH_FORM = text(
+    str(_MAIN_QUERY).replace("s.has_sixth_form,", "NULL AS has_sixth_form,")
+)
+assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), (
+    "expected replacement of 's.has_sixth_form,' to have taken effect"
+)
+
 
 def load_school_data_as_dataframe() -> pd.DataFrame:
     """Load all school + KS2 data as a pandas DataFrame."""
     try:
         df = pd.read_sql(_MAIN_QUERY, engine)
+    except sqlalchemy.exc.ProgrammingError as exc:
+        if "has_sixth_form" not in str(exc):
+            print(f"Warning: Could not load school data from marts: {exc}")
+            return pd.DataFrame()
+        logging.getLogger(__name__).warning(
+            "marts.dim_school is missing has_sixth_form (pipeline hasn't "
+            "rebuilt the mart yet on this DB) — retrying without it: %s",
+            exc,
+        )
+        try:
+            df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine)
+        except Exception as exc2:
+            print(f"Warning: Could not load school data from marts: {exc2}")
+            return pd.DataFrame()
     except Exception as exc:
         print(f"Warning: Could not load school data from marts: {exc}")
         return pd.DataFrame()

backend/tests/test_sixth_form_flag.py

@@ -95,3 +95,84 @@ def test_detail_payload_includes_flag(client):
     resp = client.get("/api/schools/100002")
     assert resp.status_code == 200, resp.text
     assert resp.json()["school_info"]["has_sixth_form"] is True
+
+
+def test_detail_payload_serializes_numpy_bool(monkeypatch):
+    """Once the pipeline has run, has_sixth_form is a real bool dtype column
+    (dbt not_null test guarantees no NULLs), so row access yields
+    numpy.bool_ rather than a Python bool. convert_to_native must handle it —
+    otherwise FastAPI's jsonable_encoder raises ValueError and the detail
+    endpoint 500s (C2)."""
+    from backend import app as app_module
+
+    df = _schools_df()
+    # Drop the row with a None flag — this fixture models the post-pipeline
+    # state where the column is a genuine, fully-populated bool dtype.
+    df = df[df["has_sixth_form"].notna()].reset_index(drop=True)
+    df["has_sixth_form"] = df["has_sixth_form"].astype(bool)
+    assert df["has_sixth_form"].dtype == bool
+
+    monkeypatch.setattr(app_module, "load_school_data", lambda: df)
+    monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
+    client = TestClient(app_module.app, raise_server_exceptions=False)
+
+    resp = client.get("/api/schools/100002")
+    assert resp.status_code == 200, resp.text
+    assert resp.json()["school_info"]["has_sixth_form"] is True
+
+
+class _FakeProgrammingError(Exception):
+    """Stand-in for sqlalchemy.exc.ProgrammingError wrapping psycopg2's
+    UndefinedColumn, without needing a real DB connection to construct one."""
+
+
+def test_load_school_data_survives_missing_has_sixth_form_column(monkeypatch):
+    """Real prod state until the nightly pipeline first rebuilds the mart:
+    marts.dim_school lacks has_sixth_form entirely. The first query raises
+    UndefinedColumn; load_school_data_as_dataframe must retry without the
+    column (synthesizing it as None) rather than swallow the error and
+    return (and then have load_school_data cache) an empty DataFrame (C1)."""
+    import sqlalchemy.exc
+    from backend import data_loader
+
+    data_loader._df_cache = None
+    data_loader._df_latest_cache = None
+
+    good_df = pd.DataFrame(
+        [
+            {
+                "urn": 1,
+                "school_name": "Fallback School",
+                "school_type": "Academy",
+                "has_sixth_form": None,
+            }
+        ]
+    )
+    calls = []
+
+    def fake_read_sql(query, con):
+        calls.append(query)
+        if len(calls) == 1:
+            raise sqlalchemy.exc.ProgrammingError(
+                "SELECT ...",
+                None,
+                Exception(
+                    "(psycopg2.errors.UndefinedColumn) column s.has_sixth_form "
+                    "does not exist"
+                ),
+            )
+        return good_df.copy()
+
+    monkeypatch.setattr(data_loader.pd, "read_sql", fake_read_sql)
+
+    try:
+        df = data_loader.load_school_data_as_dataframe()
+    finally:
+        data_loader._df_cache = None
+        data_loader._df_latest_cache = None
+
+    assert len(calls) == 2, "must retry with the no-sixth-form query variant"
+    assert calls[1] is data_loader._MAIN_QUERY_NO_SIXTH_FORM
+    assert not df.empty
+    assert "has_sixth_form" in df.columns
+    assert df["has_sixth_form"].iloc[0] is None

backend/utils.py

@@ -11,6 +11,8 @@ def convert_to_native(value: Any) -> Any:
     """Convert numpy types to native Python types for JSON serialization."""
     if pd.isna(value):
         return None
+    if isinstance(value, np.bool_):
+        return bool(value)
     if isinstance(value, (np.integer,)):
         return int(value)
     if isinstance(value, (np.floating,)):