feat(pipeline): carry the report-card inspection's own date; pick newest MI file in discovery

The MI file's report-card grade columns belong to the latest FULL
inspection (col 'Inspection start date'), but inspection_date maps to the
legacy OEIF graded/ungraded dates — so report cards were being dated with
pre-Nov-2025 inspections. Also discover_csv_url() returned matches[0],
the oldest (2017) link on the GOV.UK page.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB
This commit is contained in:
Tudor
2026-07-16 14:44:51 +01:00
co-authored by Claude Fable 5
parent 026a7ab6aa
commit e00a1b38a8
4 changed files with 59 additions and 6 deletions
@@ -34,6 +34,7 @@ select
rc_leadership_governance,
rc_early_years,
rc_sixth_form,
rc_inspection_date,
report_url
from ranked
where rn = 1
@@ -23,5 +23,6 @@ select
rc_leadership_governance,
rc_early_years,
rc_sixth_form,
rc_inspection_date,
report_url
from {{ ref('stg_ofsted_inspections') }}
@@ -49,6 +49,12 @@ renamed as (
{{ parse_report_card_grade('rc_early_years') }}::integer as rc_early_years,
{{ parse_report_card_grade('rc_sixth_form') }}::integer as rc_sixth_form,
-- Start date of the latest FULL inspection (the report-card
-- inspection in the renewed framework). Guarded in the final select:
-- only kept when the row actually carries report-card grades, because
-- in legacy-format files this column is the legacy inspection date.
to_date(nullif(trim(rc_inspection_date), 'NULL'), 'DD/MM/YYYY') as rc_inspection_date_raw,
nullif(trim(report_url), 'NULL') as report_url
from source
where urn is not null
@@ -58,5 +64,17 @@ renamed as (
)
)
select * from renamed
select
*,
case
when rc_safeguarding_met is not null
or rc_inclusion is not null
or rc_curriculum_teaching is not null
or rc_achievement is not null
or rc_attendance_behaviour is not null
or rc_personal_development is not null
or rc_leadership_governance is not null
then rc_inspection_date_raw
end as rc_inspection_date
from renamed
where inspection_date is not null