tudor/school_compare
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pipeline): ingest GIAS code columns; staging exposes codes not names

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tudor
2026-07-09 10:41:52 +01:00
co-authored by Claude Fable 5
parent e188c2ff4b
commit d898e6279b
2 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py
+6
View File
@@ -31,16 +31,22 @@ class GIASEstablishmentsStream(Stream):
schema = th.PropertiesList( schema = th.PropertiesList(
th.Property("URN", th.IntegerType, required=True), th.Property("URN", th.IntegerType, required=True),
th.Property("EstablishmentName", th.StringType), th.Property("EstablishmentName", th.StringType),
th.Property("TypeOfEstablishment (code)", th.StringType),
th.Property("TypeOfEstablishment (name)", th.StringType), th.Property("TypeOfEstablishment (name)", th.StringType),
th.Property("PhaseOfEducation (code)", th.StringType),
th.Property("PhaseOfEducation (name)", th.StringType), th.Property("PhaseOfEducation (name)", th.StringType),
th.Property("OfficialSixthForm (code)", th.StringType),
th.Property("OfficialSixthForm (name)", th.StringType), th.Property("OfficialSixthForm (name)", th.StringType),
th.Property("LA (code)", th.StringType), th.Property("LA (code)", th.StringType),
th.Property("LA (name)", th.StringType), th.Property("LA (name)", th.StringType),
th.Property("EstablishmentNumber", th.StringType), th.Property("EstablishmentNumber", th.StringType),
th.Property("EstablishmentStatus (code)", th.StringType),
th.Property("EstablishmentStatus (name)", th.StringType), th.Property("EstablishmentStatus (name)", th.StringType),
th.Property("Postcode", th.StringType), th.Property("Postcode", th.StringType),
th.Property("Gender (name)", th.StringType), th.Property("Gender (name)", th.StringType),
th.Property("ReligiousCharacter (code)", th.StringType),
th.Property("ReligiousCharacter (name)", th.StringType), th.Property("ReligiousCharacter (name)", th.StringType),
th.Property("AdmissionsPolicy (code)", th.StringType),
th.Property("AdmissionsPolicy (name)", th.StringType), th.Property("AdmissionsPolicy (name)", th.StringType),
th.Property("SchoolCapacity", th.StringType), th.Property("SchoolCapacity", th.StringType),
th.Property("NumberOfPupils", th.StringType), th.Property("NumberOfPupils", th.StringType),
pipeline/transform/models/staging/stg_gias_establishments.sql
+6 -6
View File
@@ -12,12 +12,12 @@ renamed as (
"LA (name)" as local_authority_name, "LA (name)" as local_authority_name,
cast(nullif("EstablishmentNumber", '') as integer) as establishment_number, cast(nullif("EstablishmentNumber", '') as integer) as establishment_number,
"EstablishmentName" as school_name, "EstablishmentName" as school_name,
"TypeOfEstablishment (name)" as school_type, cast(nullif(trim("TypeOfEstablishment (code)"), '') as integer) as school_type_code,
"PhaseOfEducation (name)" as phase, cast(nullif(trim("PhaseOfEducation (code)"), '') as integer) as phase_code,
nullif(trim("OfficialSixthForm (name)"), '') as official_sixth_form, cast(nullif(trim("OfficialSixthForm (code)"), '') as integer) as official_sixth_form_code,
"Gender (name)" as gender, "Gender (name)" as gender,
"ReligiousCharacter (name)" as religious_character, cast(nullif(trim("ReligiousCharacter (code)"), '') as integer) as religious_character_code,
"AdmissionsPolicy (name)" as admissions_policy, cast(nullif(trim("AdmissionsPolicy (code)"), '') as integer) as admissions_policy_code,
"SchoolCapacity" as capacity, "SchoolCapacity" as capacity,
cast(nullif("NumberOfPupils", '') as integer) as total_pupils, cast(nullif("NumberOfPupils", '') as integer) as total_pupils,
"HeadTitle (name)" as head_title, "HeadTitle (name)" as head_title,
@@ -30,7 +30,7 @@ renamed as (
"Town" as town, "Town" as town,
"County (name)" as county, "County (name)" as county,
"Postcode" as postcode, "Postcode" as postcode,
"EstablishmentStatus (name)" as status, cast(nullif(trim("EstablishmentStatus (code)"), '') as integer) as status_code,
case when "OpenDate" = '' then null else to_date("OpenDate", 'DD-MM-YYYY') end as open_date, case when "OpenDate" = '' then null else to_date("OpenDate", 'DD-MM-YYYY') end as open_date,
case when "CloseDate" = '' then null else to_date("CloseDate", 'DD-MM-YYYY') end as close_date, case when "CloseDate" = '' then null else to_date("CloseDate", 'DD-MM-YYYY') end as close_date,
"Trusts (name)" as academy_trust_name, "Trusts (name)" as academy_trust_name,