TudorandClaude Fable 5
a524cdc591
- data_loader.load_school_data_as_dataframe now catches a ProgrammingError
whose message mentions has_sixth_form (psycopg2 UndefinedColumn) and
retries with a NULL-AS-has_sixth_form query variant, so the API keeps
serving data (and the app.py column-fallback branch stays reachable)
even before the nightly pipeline has rebuilt marts.dim_school.
- utils.convert_to_native now handles numpy.bool_ so GET /api/schools/{urn}
doesn't 500 once has_sixth_form is a populated bool-dtype column.
- Update the now-stale comment on the app.py age-range fallback branch.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
40 lines
1.1 KiB
Python
40 lines
1.1 KiB
Python
"""
|
|
Utility functions for data conversion and JSON serialization.
|
|
"""
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
from typing import Any, List
|
|
|
|
|
|
def convert_to_native(value: Any) -> Any:
|
|
"""Convert numpy types to native Python types for JSON serialization."""
|
|
if pd.isna(value):
|
|
return None
|
|
if isinstance(value, np.bool_):
|
|
return bool(value)
|
|
if isinstance(value, (np.integer,)):
|
|
return int(value)
|
|
if isinstance(value, (np.floating,)):
|
|
if np.isnan(value) or np.isinf(value):
|
|
return None
|
|
return float(value)
|
|
if isinstance(value, np.ndarray):
|
|
return value.tolist()
|
|
if value == "SUPP" or value == "NE" or value == "NA" or value == "NP":
|
|
return None
|
|
return value
|
|
|
|
|
|
def clean_for_json(df: pd.DataFrame) -> List[dict]:
|
|
"""Convert DataFrame to list of dicts, replacing NaN/inf with None for JSON serialization."""
|
|
records = df.to_dict(orient="records")
|
|
cleaned = []
|
|
for record in records:
|
|
clean_record = {}
|
|
for key, value in record.items():
|
|
clean_record[key] = convert_to_native(value)
|
|
cleaned.append(clean_record)
|
|
return cleaned
|
|
|