From 95f10bf3522617551fd3b4507a6f7d3929028cf1 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 16 Jul 2026 08:29:59 +0100 Subject: [PATCH] fix: convert NaN/NULL to None and restore record properties structure in tap.py --- .../plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index d58d7b5..2c3471a 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -237,18 +237,15 @@ class OfstedInspectionsStream(Stream): for _, row in df.iterrows(): record = {} - for key in self.schema["properties"].keys(): - record[key] = None - for field, col in col_map.items(): val = row.get(col, None) - if val == 'NULL': + if pd.isna(val) or val == 'NULL': val = None record[field] = val # Cast URN try: - record["urn"] = int(record["urn"]) + record["urn"] = int(record.get("urn")) except (ValueError, KeyError, TypeError): continue