diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 68a683e..39dc868 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -111,7 +111,7 @@ print(f'Validation passed: {{count}} GIAS rows') # where that DAG hasn't run (pipeline/tests/test_dag_selectors.py). dbt_build = BashOperator( task_id="dbt_build", - bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_gias_establishments+ stg_gias_links+ gias_code_names+ --exclude int_ks2_with_lineage+ int_ks4_with_lineage+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+", + bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_gias_establishments+ stg_gias_links+ gias_code_names+ stg_gias_ukprn --exclude int_ks2_with_lineage+ int_ks4_with_lineage+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+", ) sync_typesense = BashOperator( diff --git a/pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py b/pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py index 4a217a0..c35ffdb 100644 --- a/pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py +++ b/pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py @@ -72,6 +72,12 @@ class GIASEstablishmentsStream(Stream): th.Property("Northing", th.StringType), th.Property("StatutoryLowAge", th.StringType), th.Property("StatutoryHighAge", th.StringType), + # Matches Ofsted's further-education providers, which some FE + # records name by UKPRN only (audit H4). + th.Property("UKPRN", th.StringType), + # "ISI" for independent schools the Independent Schools Inspectorate + # inspects; "Ofsted" or "Not applicable" otherwise (audit H4). + th.Property("InspectorateName (name)", th.StringType), ).to_dict() def get_records(self, context): diff --git a/pipeline/tests/test_gias_declared_fields.py b/pipeline/tests/test_gias_declared_fields.py new file mode 100644 index 0000000..bd693fa --- /dev/null +++ b/pipeline/tests/test_gias_declared_fields.py @@ -0,0 +1,61 @@ +"""Every field the GIAS establishments stream declares must be a column of the +extract GIAS publishes, or it silently loads as null. H4 adds UKPRN (matches +Ofsted's further-education providers) and InspectorateName (name) ("ISI" for +about 1,340 independent schools). HEADER is GIAS's own, from +edubasealldata20261013.csv. +""" +import re +from pathlib import Path + +TAP = (Path(__file__).resolve().parents[1] / 'plugins' / 'extractors' / 'tap-uk-gias' + / 'tap_uk_gias' / 'tap.py') + +HEADER = ( + 'URN', 'LA (code)', 'LA (name)', 'EstablishmentNumber', 'EstablishmentName', + 'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)', 'EstablishmentTypeGroup (code)', + 'EstablishmentTypeGroup (name)', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)', + 'ReasonEstablishmentOpened (code)', 'ReasonEstablishmentOpened (name)', 'OpenDate', + 'ReasonEstablishmentClosed (code)', 'ReasonEstablishmentClosed (name)', 'CloseDate', + 'PhaseOfEducation (code)', 'PhaseOfEducation (name)', 'StatutoryLowAge', 'StatutoryHighAge', + 'Boarders (code)', 'Boarders (name)', 'NurseryProvision (name)', 'OfficialSixthForm (code)', + 'OfficialSixthForm (name)', 'Gender (code)', 'Gender (name)', 'ReligiousCharacter (code)', + 'ReligiousCharacter (name)', 'ReligiousEthos (name)', 'Diocese (code)', 'Diocese (name)', + 'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)', 'SchoolCapacity', 'SpecialClasses (code)', + 'SpecialClasses (name)', 'CensusDate', 'NumberOfPupils', 'NumberOfBoys', 'NumberOfGirls', + 'PercentageFSM', 'TrustSchoolFlag (code)', 'TrustSchoolFlag (name)', 'Trusts (code)', + 'Trusts (name)', 'SchoolSponsorFlag (name)', 'SchoolSponsors (name)', 'FederationFlag (name)', + 'Federations (code)', 'Federations (name)', 'UKPRN', 'FEHEIdentifier', + 'FurtherEducationType (name)', 'LastChangedDate', 'Street', 'Locality', 'Address3', 'Town', + 'County (name)', 'Postcode', 'SchoolWebsite', 'TelephoneNum', 'HeadTitle (name)', + 'HeadFirstName', 'HeadLastName', 'HeadPreferredJobTitle', 'BSOInspectorateName (name)', + 'InspectorateReport', 'DateOfLastInspectionVisit', 'NextInspectionVisit', 'TeenMoth (name)', + 'TeenMothPlaces', 'CCF (name)', 'SENPRU (name)', 'EBD (name)', 'PlacesPRU', 'FTProv (name)', + 'EdByOther (name)', 'Section41Approved (name)', 'SEN1 (name)', 'SEN2 (name)', 'SEN3 (name)', + 'SEN4 (name)', 'SEN5 (name)', 'SEN6 (name)', 'SEN7 (name)', 'SEN8 (name)', 'SEN9 (name)', + 'SEN10 (name)', 'SEN11 (name)', 'SEN12 (name)', 'SEN13 (name)', + 'TypeOfResourcedProvision (name)', 'ResourcedProvisionOnRoll', 'ResourcedProvisionCapacity', + 'SenUnitOnRoll', 'SenUnitCapacity', 'GOR (code)', 'GOR (name)', + 'DistrictAdministrative (code)', 'DistrictAdministrative (name)', 'AdministrativeWard (code)', + 'AdministrativeWard (name)', 'ParliamentaryConstituency (code)', + 'ParliamentaryConstituency (name)', 'UrbanRural (code)', 'UrbanRural (name)', + 'GSSLACode (name)', 'Easting', 'Northing', 'MSOA (name)', 'LSOA (name)', + 'InspectorateName (name)', 'SENStat', 'SENNoStat', 'BoardingEstablishment (name)', 'PropsName', + 'PreviousLA (code)', 'PreviousLA (name)', 'PreviousEstablishmentNumber', 'Country (name)', + 'UPRN', 'SiteName', 'QABName (code)', 'QABName (name)', 'EstablishmentAccredited (code)', + 'EstablishmentAccredited (name)', 'QABReport', 'CHNumber', 'MSOA (code)', 'LSOA (code)', 'FSM', + 'AccreditationExpiryDate', +) + + +def declared(): + source = TAP.read_text() + block = source[source.index('class GIASEstablishmentsStream'):source.index('class GIASLinksStream')] + return re.findall(r'th\.Property\("([^"]+)"', block) + + +def test_every_declared_field_is_a_gias_column(): + assert set(declared()) - set(HEADER) == set() + + +def test_the_inspection_fields_are_declared(): + assert {'UKPRN', 'InspectorateName (name)'} <= set(declared()) diff --git a/pipeline/transform/models/staging/stg_gias_establishments.sql b/pipeline/transform/models/staging/stg_gias_establishments.sql index 53b97ef..1db9e55 100644 --- a/pipeline/transform/models/staging/stg_gias_establishments.sql +++ b/pipeline/transform/models/staging/stg_gias_establishments.sql @@ -38,6 +38,7 @@ renamed as ( "UrbanRural (name)" as urban_rural, "ParliamentaryConstituency (name)" as parliamentary_constituency, "NurseryProvision (name)" as nursery_provision, + nullif(trim("InspectorateName (name)"), '') as inspectorate_name, cast(nullif("Easting", '') as integer) as easting, cast(nullif("Northing", '') as integer) as northing, cast(nullif("StatutoryLowAge", '') as integer) as statutory_low_age, diff --git a/pipeline/transform/models/staging/stg_gias_ukprn.sql b/pipeline/transform/models/staging/stg_gias_ukprn.sql new file mode 100644 index 0000000..640aeb8 --- /dev/null +++ b/pipeline/transform/models/staging/stg_gias_ukprn.sql @@ -0,0 +1,11 @@ +-- URN → UKPRN from the raw GIAS extract, for matching Ofsted's further +-- education providers (int_ofsted_fe_latest, int_ofsted_listed; audit H4). +-- A model on the raw source rather than a column of stg_gias_establishments: +-- a ref to that model would put the Ofsted models into the daily build's +-- stg_gias_establishments+ selection (pipeline/tests/test_dag_selectors.py). + +select + cast("URN" as integer) as urn, + nullif(trim("UKPRN"), '') as ukprn +from {{ source('raw', 'gias_establishments') }} +where "URN" ~ '^[0-9]+$'