From a102508ef16a66d806301256a3232018a869cd35 Mon Sep 17 00:00:00 2001 From: Tudor Date: Fri, 24 Jul 2026 09:58:27 +0100 Subject: [PATCH] fix(data): strip any table alias in missing-column matcher, not just s. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graceful-degradation fallback keys off the column named in a Postgres UndefinedColumn error, but the matcher only stripped an `s.` alias. The two new dim_location columns (county, parliamentary_constituency) are selected via the `l.` alias and Postgres reports them unquoted as "column l.county does not exist" — which the old regex failed to match at all, returning None. If dim_school is rebuilt (telephone/nursery present) but dim_location is not yet (county/parliamentary_constituency missing) — plausible since they are independently-rebuilt dbt models — the fallback branch never matched and load_school_data_as_dataframe() returned an empty DataFrame, showing zero schools sitewide instead of degrading those columns to NULL. Generalise the alias prefix to `\w+\.` and cover the l.-qualified case in tests. Co-Authored-By: Claude Opus 4.8 --- backend/data_loader.py | 6 +++++- backend/tests/test_gias_translation.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/backend/data_loader.py b/backend/data_loader.py index 7f039d8..e3aec2f 100644 --- a/backend/data_loader.py +++ b/backend/data_loader.py @@ -339,7 +339,11 @@ _GIAS_CODE_COLUMN_NAMES = ( "admissions_policy_code", ) -_MISSING_COLUMN_RE = re.compile(r'column "?(?:s\.)?(\w+)"? does not exist') +# Strip any single table alias prefix (s., l., p., foi., …) — Postgres reports +# an undefined *qualified* column unquoted as "column l.county does not exist", +# so matching only the s. alias would miss dim_location columns (county, +# parliamentary_constituency) and defeat the graceful-degradation fallback. +_MISSING_COLUMN_RE = re.compile(r'column "?(?:\w+\.)?(\w+)"? does not exist') def _missing_column_name(exc: Exception) -> Optional[str]: diff --git a/backend/tests/test_gias_translation.py b/backend/tests/test_gias_translation.py index 414b8e4..9b57a10 100644 --- a/backend/tests/test_gias_translation.py +++ b/backend/tests/test_gias_translation.py @@ -73,6 +73,24 @@ def test_missing_column_name_table_prefixed(): ) +def test_missing_column_name_location_alias_prefixed(): + # dim_location columns are selected via the `l.` alias; Postgres reports a + # missing qualified column unquoted (e.g. "column l.county does not exist"). + # The matcher must strip any alias, not just `s.`, or the county / + # parliamentary_constituency fallback never triggers and the whole data + # load degrades to an empty DataFrame (zero schools) instead of NULLs. + assert ( + _missing_column_name(_fake_exc("column l.county does not exist")) + == "county" + ) + assert ( + _missing_column_name( + _fake_exc("column l.parliamentary_constituency does not exist") + ) + == "parliamentary_constituency" + ) + + def test_missing_column_name_no_match_returns_none(): assert _missing_column_name(_fake_exc("relation \"marts.dim_school\" does not exist")) is None