fix(api): survive missing has_sixth_form column and numpy bool serialization
- data_loader.load_school_data_as_dataframe now catches a ProgrammingError
whose message mentions has_sixth_form (psycopg2 UndefinedColumn) and
retries with a NULL-AS-has_sixth_form query variant, so the API keeps
serving data (and the app.py column-fallback branch stays reachable)
even before the nightly pipeline has rebuilt marts.dim_school.
- utils.convert_to_native now handles numpy.bool_ so GET /api/schools/{urn}
doesn't 500 once has_sixth_form is a populated bool-dtype column.
- Update the now-stale comment on the app.py age-range fallback branch.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,11 +3,14 @@ Data loading module — reads from marts.* tables built by dbt.
|
||||
Provides efficient queries with caching.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional, Dict, Tuple, List
|
||||
import requests
|
||||
from sqlalchemy import text
|
||||
import sqlalchemy.exc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import settings
|
||||
@@ -215,11 +218,36 @@ _MAIN_QUERY = text("""
|
||||
ORDER BY s.school_name, p.year
|
||||
""")
|
||||
|
||||
# Fallback used when marts.dim_school predates the has_sixth_form column
|
||||
# (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on this DB).
|
||||
# Keeps the column present as NULL so downstream code — including the
|
||||
# app.py fallback branch — behaves as designed instead of KeyError-ing.
|
||||
_MAIN_QUERY_NO_SIXTH_FORM = text(
|
||||
str(_MAIN_QUERY).replace("s.has_sixth_form,", "NULL AS has_sixth_form,")
|
||||
)
|
||||
assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), (
|
||||
"expected replacement of 's.has_sixth_form,' to have taken effect"
|
||||
)
|
||||
|
||||
|
||||
def load_school_data_as_dataframe() -> pd.DataFrame:
|
||||
"""Load all school + KS2 data as a pandas DataFrame."""
|
||||
try:
|
||||
df = pd.read_sql(_MAIN_QUERY, engine)
|
||||
except sqlalchemy.exc.ProgrammingError as exc:
|
||||
if "has_sixth_form" not in str(exc):
|
||||
print(f"Warning: Could not load school data from marts: {exc}")
|
||||
return pd.DataFrame()
|
||||
logging.getLogger(__name__).warning(
|
||||
"marts.dim_school is missing has_sixth_form (pipeline hasn't "
|
||||
"rebuilt the mart yet on this DB) — retrying without it: %s",
|
||||
exc,
|
||||
)
|
||||
try:
|
||||
df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine)
|
||||
except Exception as exc2:
|
||||
print(f"Warning: Could not load school data from marts: {exc2}")
|
||||
return pd.DataFrame()
|
||||
except Exception as exc:
|
||||
print(f"Warning: Could not load school data from marts: {exc}")
|
||||
return pd.DataFrame()
|
||||
|
||||
Reference in New Issue
Block a user