From 719f06e480018bd75c2d09f7ccd9e79806e7b5bb Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 26 Mar 2026 22:45:02 +0000 Subject: [PATCH] fix(pipeline): make total_pupils non-optional for Typesense, add lat/lng to dim_location - Remove optional flag from total_pupils (Typesense requires default sorting field to be non-optional) - Add latitude/longitude columns to dim_location computed from PostGIS geom, for direct use by backend and Typesense sync Co-Authored-By: Claude Opus 4.6 --- pipeline/scripts/sync_typesense.py | 6 +++--- pipeline/transform/models/marts/dim_location.sql | 12 +++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pipeline/scripts/sync_typesense.py b/pipeline/scripts/sync_typesense.py index cce0660..30194c5 100644 --- a/pipeline/scripts/sync_typesense.py +++ b/pipeline/scripts/sync_typesense.py @@ -33,7 +33,7 @@ COLLECTION_SCHEMA = { {"name": "headteacher_name", "type": "string", "optional": True}, {"name": "rwm_expected_pct", "type": "float", "optional": True}, {"name": "progress_8_score", "type": "float", "optional": True}, - {"name": "total_pupils", "type": "int32", "optional": True}, + {"name": "total_pupils", "type": "int32"}, ], "default_sorting_field": "total_pupils", } @@ -52,8 +52,8 @@ QUERY_BASE = """ l.postcode, s.headteacher_name, s.total_pupils, - ST_Y(l.geom) as lat, - ST_X(l.geom) as lng + l.latitude as lat, + l.longitude as lng FROM marts.dim_school s LEFT JOIN marts.dim_location l ON s.urn = l.urn """ diff --git a/pipeline/transform/models/marts/dim_location.sql b/pipeline/transform/models/marts/dim_location.sql index 11fc951..be285a1 100644 --- a/pipeline/transform/models/marts/dim_location.sql +++ b/pipeline/transform/models/marts/dim_location.sql @@ -19,6 +19,16 @@ select when s.easting is not null and s.northing is not null then ST_Transform(ST_SetSRID(ST_MakePoint(s.easting::double precision, s.northing::double precision), 27700), 4326) else null - end as geom + end as geom, + case + when s.easting is not null and s.northing is not null + then ST_Y(ST_Transform(ST_SetSRID(ST_MakePoint(s.easting::double precision, s.northing::double precision), 27700), 4326)) + else null + end as latitude, + case + when s.easting is not null and s.northing is not null + then ST_X(ST_Transform(ST_SetSRID(ST_MakePoint(s.easting::double precision, s.northing::double precision), 27700), 4326)) + else null + end as longitude from {{ ref('stg_gias_establishments') }} s where s.status = 'Open'