fix(api): survive missing has_sixth_form column and numpy bool serialization

- data_loader.load_school_data_as_dataframe now catches a ProgrammingError
  whose message mentions has_sixth_form (psycopg2 UndefinedColumn) and
  retries with a NULL-AS-has_sixth_form query variant, so the API keeps
  serving data (and the app.py column-fallback branch stays reachable)
  even before the nightly pipeline has rebuilt marts.dim_school.
- utils.convert_to_native now handles numpy.bool_ so GET /api/schools/{urn}
  doesn't 500 once has_sixth_form is a populated bool-dtype column.
- Update the now-stale comment on the app.py age-range fallback branch.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tudor
2026-07-07 13:33:25 +01:00
co-authored by Claude Fable 5
parent 3fcb1340d4
commit a524cdc591
4 changed files with 115 additions and 1 deletions
+28
View File
@@ -3,11 +3,14 @@ Data loading module — reads from marts.* tables built by dbt.
Provides efficient queries with caching.
"""
import logging
import pandas as pd
import numpy as np
from typing import Optional, Dict, Tuple, List
import requests
from sqlalchemy import text
import sqlalchemy.exc
from sqlalchemy.orm import Session
from .config import settings
@@ -215,11 +218,36 @@ _MAIN_QUERY = text("""
ORDER BY s.school_name, p.year
""")
# Fallback used when marts.dim_school predates the has_sixth_form column
# (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on this DB).
# Keeps the column present as NULL so downstream code — including the
# app.py fallback branch — behaves as designed instead of KeyError-ing.
_MAIN_QUERY_NO_SIXTH_FORM = text(
str(_MAIN_QUERY).replace("s.has_sixth_form,", "NULL AS has_sixth_form,")
)
assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), (
"expected replacement of 's.has_sixth_form,' to have taken effect"
)
def load_school_data_as_dataframe() -> pd.DataFrame:
"""Load all school + KS2 data as a pandas DataFrame."""
try:
df = pd.read_sql(_MAIN_QUERY, engine)
except sqlalchemy.exc.ProgrammingError as exc:
if "has_sixth_form" not in str(exc):
print(f"Warning: Could not load school data from marts: {exc}")
return pd.DataFrame()
logging.getLogger(__name__).warning(
"marts.dim_school is missing has_sixth_form (pipeline hasn't "
"rebuilt the mart yet on this DB) — retrying without it: %s",
exc,
)
try:
df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine)
except Exception as exc2:
print(f"Warning: Could not load school data from marts: {exc2}")
return pd.DataFrame()
except Exception as exc:
print(f"Warning: Could not load school data from marts: {exc}")
return pd.DataFrame()