From c5645664327ff1ab7f7abcdfe4b647bf582a7e39 Mon Sep 17 00:00:00 2001 From: Tudor Date: Fri, 28 Aug 2026 16:07:17 +0100 Subject: [PATCH] feat(destinations): staging models that keep 'withheld' distinct from 'absent' safe_numeric maps every EES sentinel to NULL, which is right for attainment and wrong here: one of those states has to print 'withheld' and the other has to print nothing. A status column carries the difference. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BvdDKvFFSZuMVDH5fEyTob --- .../transform/models/staging/_stg_sources.yml | 14 ++++++ .../staging/stg_ees_ks4_destinations.sql | 46 ++++++++++++++++++ .../staging/stg_ees_ks5_destinations.sql | 47 +++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 pipeline/transform/models/staging/stg_ees_ks4_destinations.sql create mode 100644 pipeline/transform/models/staging/stg_ees_ks5_destinations.sql diff --git a/pipeline/transform/models/staging/_stg_sources.yml b/pipeline/transform/models/staging/_stg_sources.yml index 6b294c4..564927b 100644 --- a/pipeline/transform/models/staging/_stg_sources.yml +++ b/pipeline/transform/models/staging/_stg_sources.yml @@ -33,6 +33,20 @@ sources: - name: legacy_ks2 description: Pre-COVID KS2 school-level data (2015-16 to 2018-19) from DfE performance tables + - name: ees_ks4_destinations + description: > + KS4 leavers destinations, long format — one row per URN x year x + pupil group x destination measure, plus England rows with a null urn. + pupils_raw and percentage_raw are TEXT and may carry the 'c' + suppression sentinel; they must never be passed through safe_numeric, + because the categories sum to the cohort and collapsing 'c' to null + lets a downstream sum recover a withheld figure. + + - name: ees_ks5_destinations + description: > + 16-18 study leavers destinations. Same grain, same suppression + caveat, and only institutions with post-16 provision appear. + - name: ees_ks4_performance description: KS4 performance tables (long format — one row per school × breakdown × sex) diff --git a/pipeline/transform/models/staging/stg_ees_ks4_destinations.sql b/pipeline/transform/models/staging/stg_ees_ks4_destinations.sql new file mode 100644 index 0000000..95d1ca4 --- /dev/null +++ b/pipeline/transform/models/staging/stg_ees_ks4_destinations.sql @@ -0,0 +1,46 @@ +{{ config(materialized='table') }} + +-- Staging model: KS4 leavers destinations, school level plus the England +-- reference (which carries a null urn). +-- +-- DELIBERATELY DOES NOT USE safe_numeric. That macro maps every EES sentinel +-- (z, c, x, q, u) to NULL, which is right for attainment — there, "suppressed" +-- and "not applicable" are equally unrenderable. Here they are different +-- claims: one has to print "withheld", the other has to print nothing, and +-- collapsing them would also let a downstream sum reconstruct a withheld +-- figure, because the destination categories add up to the cohort. +-- +-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md. + +with source as ( + select * from {{ source('raw', 'ees_ks4_destinations') }} + -- National rows carry a null urn and feed fact_destination_national. + where (urn is null or urn = '' or urn ~ '^[0-9]+$') + and time_period ~ '^[0-9]+$' +) + +select + case when urn ~ '^[0-9]+$' then cast(trim(urn) as integer) end as urn, + cast(trim(time_period) as integer) as year, + trim(pupil_group) as pupil_group, + trim(destination_measure) as destination_measure, + + case when cohort_pupils ~ '^[0-9]+$' + then cast(cohort_pupils as integer) end as cohort_pupils, + + case when pupils_raw ~ '^[0-9]+$' + then cast(pupils_raw as integer) end as pupils, + + case when percentage_raw ~ '^-?[0-9]+(\.[0-9]+)?$' + then cast(percentage_raw as numeric) end as percentage, + + -- The whole point of this model. 'suppressed' is DfE withholding a figure + -- it considers disclosive; 'not_applicable' is the measure not applying to + -- this row at all. The API and the frontend treat them differently. + case + when pupils_raw ~ '^[0-9]+$' then 'published' + when lower(trim(pupils_raw)) = 'c' then 'suppressed' + else 'not_applicable' + end as status + +from source diff --git a/pipeline/transform/models/staging/stg_ees_ks5_destinations.sql b/pipeline/transform/models/staging/stg_ees_ks5_destinations.sql new file mode 100644 index 0000000..bce4202 --- /dev/null +++ b/pipeline/transform/models/staging/stg_ees_ks5_destinations.sql @@ -0,0 +1,47 @@ +{{ config(materialized='table') }} + +-- Staging model: 16-18 study leavers destinations, institution level plus the +-- England reference (which carries a null urn). Only sixth forms and colleges +-- appear here, so a secondary with no post-16 provision has no rows at all. +-- +-- DELIBERATELY DOES NOT USE safe_numeric. That macro maps every EES sentinel +-- (z, c, x, q, u) to NULL, which is right for attainment — there, "suppressed" +-- and "not applicable" are equally unrenderable. Here they are different +-- claims: one has to print "withheld", the other has to print nothing, and +-- collapsing them would also let a downstream sum reconstruct a withheld +-- figure, because the destination categories add up to the cohort. +-- +-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md. + +with source as ( + select * from {{ source('raw', 'ees_ks5_destinations') }} + -- National rows carry a null urn and feed fact_destination_national. + where (urn is null or urn = '' or urn ~ '^[0-9]+$') + and time_period ~ '^[0-9]+$' +) + +select + case when urn ~ '^[0-9]+$' then cast(trim(urn) as integer) end as urn, + cast(trim(time_period) as integer) as year, + trim(pupil_group) as pupil_group, + trim(destination_measure) as destination_measure, + + case when cohort_pupils ~ '^[0-9]+$' + then cast(cohort_pupils as integer) end as cohort_pupils, + + case when pupils_raw ~ '^[0-9]+$' + then cast(pupils_raw as integer) end as pupils, + + case when percentage_raw ~ '^-?[0-9]+(\.[0-9]+)?$' + then cast(percentage_raw as numeric) end as percentage, + + -- The whole point of this model. 'suppressed' is DfE withholding a figure + -- it considers disclosive; 'not_applicable' is the measure not applying to + -- this row at all. The API and the frontend treat them differently. + case + when pupils_raw ~ '^[0-9]+$' then 'published' + when lower(trim(pupils_raw)) = 'c' then 'suppressed' + else 'not_applicable' + end as status + +from source