diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index d58d7b5..2c3471a 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -237,18 +237,15 @@ class OfstedInspectionsStream(Stream): for _, row in df.iterrows(): record = {} - for key in self.schema["properties"].keys(): - record[key] = None - for field, col in col_map.items(): val = row.get(col, None) - if val == 'NULL': + if pd.isna(val) or val == 'NULL': val = None record[field] = val # Cast URN try: - record["urn"] = int(record["urn"]) + record["urn"] = int(record.get("urn")) except (ValueError, KeyError, TypeError): continue