diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 1aa86c1..89f2301 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -180,6 +180,74 @@ models: - name: year tests: [not_null] + - name: fact_ks4_destinations + description: > + KS4 leavers destinations — one row per URN, year, pupil group and + destination measure. status distinguishes a figure DfE withheld + ('suppressed') from one that does not apply ('not_applicable'); pupils is + null for both, so status is the only thing that tells them apart. R3 + masking is applied here: a category withheld for disadvantaged pupils is + withheld for the other-pupils group too. + columns: + - name: urn + tests: [not_null] + - name: year + tests: [not_null] + - name: pupil_group + tests: + - not_null + - accepted_values: + values: ['all', 'disadvantaged', 'other'] + - name: destination_measure + tests: [not_null] + - name: status + tests: + - not_null + - accepted_values: + values: ['published', 'suppressed', 'not_applicable'] + + - name: fact_ks5_destinations + description: > + 16-18 study leavers destinations — same grain and same suppression + semantics as fact_ks4_destinations. Only institutions with post-16 + provision appear. + columns: + - name: urn + tests: [not_null] + - name: year + tests: [not_null] + - name: pupil_group + tests: + - not_null + - accepted_values: + values: ['all', 'disadvantaged', 'other'] + - name: destination_measure + tests: [not_null] + - name: status + tests: + - not_null + - accepted_values: + values: ['published', 'suppressed', 'not_applicable'] + + - name: fact_destination_national + description: > + England destination measures by phase and pupil group — the national + reference the school sections compare against. Repointed with the cohort + switch so a school's disadvantaged pupils are measured against national + disadvantaged pupils, not against the national average. + columns: + - name: phase + tests: + - not_null + - accepted_values: + values: ['ks4', 'ks5'] + - name: year + tests: [not_null] + - name: pupil_group + tests: [not_null] + - name: destination_measure + tests: [not_null] + - name: fact_ks2_national_averages description: Official DfE KS2 national headline averages — one row per academic year columns: diff --git a/pipeline/transform/models/marts/fact_destination_national.sql b/pipeline/transform/models/marts/fact_destination_national.sql new file mode 100644 index 0000000..811078f --- /dev/null +++ b/pipeline/transform/models/marts/fact_destination_national.sql @@ -0,0 +1,40 @@ +{{ config(materialized='table') }} + +-- Mart: England destination measures by pupil group — the page's national +-- reference. +-- +-- Kept separate from the school facts so the section's England bar can repoint +-- with the cohort switch. Comparing a school's disadvantaged pupils against +-- the national all-pupils figure would flatter or damn the school for its +-- intake rather than its work, which is the failure this whole feature is +-- meant to avoid. +-- +-- The tap pins establishment type to Total within the state-funded mainstream +-- group for these rows, so the parts reconcile: for 2022/23 KS4, 151,912 +-- disadvantaged + 441,823 other = 593,735 total. + +select + 'ks4' as phase, + year, + pupil_group, + destination_measure, + cohort_pupils, + pupils, + percentage, + status +from {{ ref('stg_ees_ks4_destinations') }} +where urn is null + +union all + +select + 'ks5' as phase, + year, + pupil_group, + destination_measure, + cohort_pupils, + pupils, + percentage, + status +from {{ ref('stg_ees_ks5_destinations') }} +where urn is null diff --git a/pipeline/transform/models/marts/fact_ks4_destinations.sql b/pipeline/transform/models/marts/fact_ks4_destinations.sql new file mode 100644 index 0000000..cf03b7c --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks4_destinations.sql @@ -0,0 +1,50 @@ +{{ config(materialized='table') }} + +-- Mart: KS4 leavers destinations — one row per URN x year x pupil group x +-- destination measure. +-- +-- Long format, unlike the wide fact_ks4_performance next door. pupil_group is +-- a real third dimension, so going wide would need three sets of every column, +-- and the disclosure tests are far easier to write over rows than columns. +-- +-- R3 IS APPLIED HERE rather than in the API. Disadvantaged and other-pupils +-- partition the whole, and the all-pupils figure is published — so publishing +-- both halves recovers the suppressed one by subtraction. Where a category is +-- withheld for disadvantaged pupils it is therefore withheld for the other +-- group too, and the all-pupils view keeps it. DfE already does this in 493 of +-- 498 cases; this closes the remaining 5 so that no consumer, now or later, +-- can reach an unmasked combination. +-- +-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md. + +with staged as ( + select s.* + from {{ ref('stg_ees_ks4_destinations') }} s + inner join {{ ref('dim_school') }} d on d.urn = s.urn +), + +-- Categories withheld for disadvantaged pupils at this school and year. +masked as ( + select distinct urn, year, destination_measure + from staged + where pupil_group = 'disadvantaged' + and status = 'suppressed' +) + +select + s.urn, + s.year, + s.pupil_group, + s.destination_measure, + s.cohort_pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.pupils end as pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.percentage end as percentage, + case when m.urn is not null and s.pupil_group = 'other' + then 'suppressed' else s.status end as status +from staged s +left join masked m + on m.urn = s.urn + and m.year = s.year + and m.destination_measure = s.destination_measure diff --git a/pipeline/transform/models/marts/fact_ks5_destinations.sql b/pipeline/transform/models/marts/fact_ks5_destinations.sql new file mode 100644 index 0000000..59cbef7 --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks5_destinations.sql @@ -0,0 +1,50 @@ +{{ config(materialized='table') }} + +-- Mart: 16-18 study leavers destinations — one row per URN x year x pupil group x +-- destination measure. +-- +-- Long format, unlike the wide fact_ks4_performance next door. pupil_group is +-- a real third dimension, so going wide would need three sets of every column, +-- and the disclosure tests are far easier to write over rows than columns. +-- +-- R3 IS APPLIED HERE rather than in the API. Disadvantaged and other-pupils +-- partition the whole, and the all-pupils figure is published — so publishing +-- both halves recovers the suppressed one by subtraction. Where a category is +-- withheld for disadvantaged pupils it is therefore withheld for the other +-- group too, and the all-pupils view keeps it. DfE already does this in 493 of +-- 498 cases; this closes the remaining 5 so that no consumer, now or later, +-- can reach an unmasked combination. +-- +-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md. + +with staged as ( + select s.* + from {{ ref('stg_ees_ks5_destinations') }} s + inner join {{ ref('dim_school') }} d on d.urn = s.urn +), + +-- Categories withheld for disadvantaged pupils at this school and year. +masked as ( + select distinct urn, year, destination_measure + from staged + where pupil_group = 'disadvantaged' + and status = 'suppressed' +) + +select + s.urn, + s.year, + s.pupil_group, + s.destination_measure, + s.cohort_pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.pupils end as pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.percentage end as percentage, + case when m.urn is not null and s.pupil_group = 'other' + then 'suppressed' else s.status end as status +from staged s +left join masked m + on m.urn = s.urn + and m.year = s.year + and m.destination_measure = s.destination_measure diff --git a/pipeline/transform/tests/assert_destination_status_null_agreement.sql b/pipeline/transform/tests/assert_destination_status_null_agreement.sql new file mode 100644 index 0000000..ec5ffda --- /dev/null +++ b/pipeline/transform/tests/assert_destination_status_null_agreement.sql @@ -0,0 +1,15 @@ +-- pupils must be null wherever status is not 'published', and never null where +-- it is. This is what stops a later coalesce, or a wide-format refactor, +-- turning a withheld figure into a zero — which would read on the page as +-- "no pupils went here" rather than "we are not told". + +select + urn, + year, + pupil_group, + destination_measure, + status, + pupils +from {{ ref('fact_ks4_destinations') }} +where (status <> 'published' and pupils is not null) + or (status = 'published' and pupils is null) diff --git a/pipeline/transform/tests/assert_destinations_group_masking.sql b/pipeline/transform/tests/assert_destinations_group_masking.sql new file mode 100644 index 0000000..586c4ed --- /dev/null +++ b/pipeline/transform/tests/assert_destinations_group_masking.sql @@ -0,0 +1,20 @@ +-- R3 GUARD. Fails if a category is suppressed for disadvantaged pupils but +-- still published for the other-pupils group. +-- +-- The two groups partition the cohort and the all-pupils figure is published, +-- so publishing both halves recovers the withheld one. fact_ks4_destinations +-- masks the other group to prevent it; this asserts the masking actually held. + +select + d.urn, + d.year, + d.destination_measure +from {{ ref('fact_ks4_destinations') }} d +inner join {{ ref('fact_ks4_destinations') }} o + on o.urn = d.urn + and o.year = d.year + and o.destination_measure = d.destination_measure + and o.pupil_group = 'other' +where d.pupil_group = 'disadvantaged' + and d.status = 'suppressed' + and o.status = 'published' diff --git a/pipeline/transform/tests/assert_destinations_no_derived_remainder.sql b/pipeline/transform/tests/assert_destinations_no_derived_remainder.sql new file mode 100644 index 0000000..c8f53f7 --- /dev/null +++ b/pipeline/transform/tests/assert_destinations_no_derived_remainder.sql @@ -0,0 +1,24 @@ +{{ config(severity='warn') }} + +-- R1 TRIPWIRE. Flags a school/year/pupil group that has exactly one suppressed +-- category while its cohort total is published — the combination that lets the +-- withheld figure be recovered by subtracting the published categories. +-- +-- This is a WARN, not an error, and the distinction matters. The mart is not +-- wrong: DfE publishes exactly this, and the mart's job is to carry the source +-- faithfully. What must refuse to close the gap is everything downstream — the +-- API serialiser and lib/destinations.ts, which have their own tests. This +-- query exists so the condition stays visible and counted, and so anyone +-- adding a consumer later has to look at it rather than discover it. +-- +-- Expect a non-trivial count: measured at 22% of mainstream secondaries. + +select + urn, + year, + pupil_group, + count(*) filter (where status = 'suppressed') as suppressed_categories +from {{ ref('fact_ks4_destinations') }} +where destination_measure not like 'agg\_%' escape '\' +group by urn, year, pupil_group +having count(*) filter (where status = 'suppressed') = 1