fix(api): match missing-column fallbacks on the DBAPI error, not the statement
PR Checks / Frontend Typecheck + Tests (pull_request) Successful in 9m41s
PR Checks / Backend Smoke (pull_request) Successful in 6s
PR Checks / Build Backend (no push) (pull_request) Successful in 20s
PR Checks / Build Frontend (no push) (pull_request) Successful in 53s
PR Checks / Build Pipeline (no push) (pull_request) Successful in 10s
PR Checks / AI Code Review (Claude) (pull_request) Successful in 2m41s

str(ProgrammingError) embeds the full SQL, which contains every column
name — the substring check matched any error and could take the wrong
retry branch. Parse the missing column from exc.orig instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tudor
2026-07-09 19:29:58 +01:00
co-authored by Claude Fable 5
parent 4b75152ee0
commit 74ca76d150
3 changed files with 65 additions and 9 deletions
+18 -2
View File
@@ -4,6 +4,7 @@ Provides efficient queries with caching.
"""
import logging
import re
import pandas as pd
import numpy as np
@@ -291,13 +292,28 @@ _GIAS_CODE_COLUMN_NAMES = (
"admissions_policy_code",
)
_MISSING_COLUMN_RE = re.compile(r'column "?(?:s\.)?(\w+)"? does not exist')
def _missing_column_name(exc: Exception) -> Optional[str]:
"""Name of the missing column from a psycopg2 UndefinedColumn error.
Inspects exc.orig (the DBAPI error), whose message names only the
offending column — str(exc) also embeds the full SQL statement, which
contains every column name and therefore must not be matched against.
"""
orig = getattr(exc, "orig", None)
match = _MISSING_COLUMN_RE.search(str(orig) if orig is not None else str(exc))
return match.group(1) if match else None
def load_school_data_as_dataframe() -> pd.DataFrame:
"""Load all school + KS2 data as a pandas DataFrame."""
try:
df = pd.read_sql(_MAIN_QUERY, engine)
except sqlalchemy.exc.ProgrammingError as exc:
if any(col in str(exc) for col in _GIAS_CODE_COLUMN_NAMES):
missing = _missing_column_name(exc)
if missing in _GIAS_CODE_COLUMN_NAMES:
logging.getLogger(__name__).warning(
"marts predate the GIAS code migration — falling back to "
"legacy name-column query: %s",
@@ -308,7 +324,7 @@ def load_school_data_as_dataframe() -> pd.DataFrame:
except Exception as exc2:
print(f"Warning: Could not load school data from marts: {exc2}")
return pd.DataFrame()
elif "has_sixth_form" in str(exc):
elif missing == "has_sixth_form":
logging.getLogger(__name__).warning(
"marts.dim_school is missing has_sixth_form (pipeline hasn't "
"rebuilt the mart yet on this DB) — retrying without it: %s",