diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py index 168d88d..eaacbd7 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py +++ b/pipeline/plugins/extractors/tap-uk-ofsted/tap_uk_ofsted/tap.py @@ -68,6 +68,19 @@ COLUMN_PRIORITY = { "ungraded_inspection_date": [ "Date of latest ungraded inspection", ], + # Report Card fields (post-Nov 2025 framework). Confirmed verbatim MI + # headers per diagnose_compare_gaps.py's Task 1(c) findings. No MI column + # currently exists for early-years or sixth-form report-card grades, so + # those two fields are deliberately omitted here (see schema below) -- + # they stay absent from every record, same as the existing `report_url` + # pattern for fields with no COLUMN_PRIORITY entry. + "rc_safeguarding_met": ["Safeguarding standards"], + "rc_inclusion": ["Inclusion"], + "rc_curriculum_teaching": ["Curriculum and teaching"], + "rc_achievement": ["Achievement"], + "rc_attendance_behaviour": ["Attendance and behaviour"], + "rc_personal_development": ["Personal development and wellbeing"], + "rc_leadership_governance": ["Leadership and governance"], } @@ -111,6 +124,17 @@ class OfstedInspectionsStream(Stream): th.Property("sixth_form_provision", th.StringType), th.Property("ungraded_outcome", th.StringType), th.Property("ungraded_inspection_date", th.StringType), + th.Property("rc_safeguarding_met", th.StringType), + th.Property("rc_inclusion", th.StringType), + th.Property("rc_curriculum_teaching", th.StringType), + th.Property("rc_achievement", th.StringType), + th.Property("rc_attendance_behaviour", th.StringType), + th.Property("rc_personal_development", th.StringType), + th.Property("rc_leadership_governance", th.StringType), + # No MI column exists for these yet; declared for forward + # compatibility with the mart schema, always emitted as absent/NULL. + th.Property("rc_early_years", th.StringType), + th.Property("rc_sixth_form", th.StringType), th.Property("report_url", th.StringType), ).to_dict() diff --git a/pipeline/transform/macros/parse_report_card_grade.sql b/pipeline/transform/macros/parse_report_card_grade.sql new file mode 100644 index 0000000..2e1bfb5 --- /dev/null +++ b/pipeline/transform/macros/parse_report_card_grade.sql @@ -0,0 +1,17 @@ +-- Macro: Parse Ofsted Report Card grade (post-Nov 2025 framework) from text +-- into the 5-point scale. Real values confirmed via a live sample of the MI +-- CSV (see pipeline/scripts/diagnose_compare_gaps.py's +-- "TASK 7 VALUE SAMPLE 2026-07-12" note) -- unrecognised text (including the +-- 'NULL' sentinel used by the source CSV for blanks) parses to NULL, never +-- errors. + +{% macro parse_report_card_grade(column_name) %} + case lower(trim(nullif({{ column_name }}, 'NULL'))) + when 'exceptional' then 1 + when 'strong standard' then 2 + when 'expected standard' then 3 + when 'needs attention' then 4 + when 'urgent improvement' then 5 + else null + end +{% endmacro %} diff --git a/pipeline/transform/models/staging/stg_ofsted_inspections.sql b/pipeline/transform/models/staging/stg_ofsted_inspections.sql index 557d1bb..95dd20a 100644 --- a/pipeline/transform/models/staging/stg_ofsted_inspections.sql +++ b/pipeline/transform/models/staging/stg_ofsted_inspections.sql @@ -33,17 +33,20 @@ renamed as ( nullif(trim(ungraded_outcome), 'NULL') as ungraded_outcome, {{ parse_ungraded_outcome('ungraded_outcome') }}::integer as ungraded_grade, - -- Report Card fields (post-Nov 2025 framework) - -- TODO: add rc_* columns to tap-uk-ofsted schema once CSV column names are confirmed - null::text as rc_safeguarding_met, - null::text as rc_inclusion, - null::text as rc_curriculum_teaching, - null::text as rc_achievement, - null::text as rc_attendance_behaviour, - null::text as rc_personal_development, - null::text as rc_leadership_governance, - null::text as rc_early_years, - null::text as rc_sixth_form, + -- Report Card fields (post-Nov 2025 framework), 5-point scale: + -- 1 Exceptional · 2 Strong standard · 3 Expected standard + -- · 4 Needs attention · 5 Urgent improvement + (lower(trim(nullif(rc_safeguarding_met, 'NULL'))) = 'met') as rc_safeguarding_met, + {{ parse_report_card_grade('rc_inclusion') }}::integer as rc_inclusion, + {{ parse_report_card_grade('rc_curriculum_teaching') }}::integer as rc_curriculum_teaching, + {{ parse_report_card_grade('rc_achievement') }}::integer as rc_achievement, + {{ parse_report_card_grade('rc_attendance_behaviour') }}::integer as rc_attendance_behaviour, + {{ parse_report_card_grade('rc_personal_development') }}::integer as rc_personal_development, + {{ parse_report_card_grade('rc_leadership_governance') }}::integer as rc_leadership_governance, + -- No MI column exists for these yet (see tap.py); the tap never + -- emits rc_early_years/rc_sixth_form, so these stay NULL. + null::integer as rc_early_years, + null::integer as rc_sixth_form, report_url from source