From 5e5b61987a44157122497150716d4bf49f55fa40 Mon Sep 17 00:00:00 2001 From: Tudor Date: Fri, 28 Aug 2026 16:08:20 +0100 Subject: [PATCH] feat(destinations): marts, with R3 masking applied at the boundary Disadvantaged and other-pupils partition the whole and the all-pupils figure is published, so publishing both halves recovers the suppressed one. The mask is applied in the mart rather than the API so no consumer added later can reach an unmasked combination. The R1 test is a warn, not an error: DfE publishes the recoverable combination and the mart's job is to carry it faithfully. Refusing to close the gap is the API's job and the frontend's. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BvdDKvFFSZuMVDH5fEyTob --- .../transform/models/marts/_marts_schema.yml | 68 +++++++++++++++++++ .../marts/fact_destination_national.sql | 40 +++++++++++ .../models/marts/fact_ks4_destinations.sql | 50 ++++++++++++++ .../models/marts/fact_ks5_destinations.sql | 50 ++++++++++++++ ...sert_destination_status_null_agreement.sql | 15 ++++ .../assert_destinations_group_masking.sql | 20 ++++++ ...sert_destinations_no_derived_remainder.sql | 24 +++++++ 7 files changed, 267 insertions(+) create mode 100644 pipeline/transform/models/marts/fact_destination_national.sql create mode 100644 pipeline/transform/models/marts/fact_ks4_destinations.sql create mode 100644 pipeline/transform/models/marts/fact_ks5_destinations.sql create mode 100644 pipeline/transform/tests/assert_destination_status_null_agreement.sql create mode 100644 pipeline/transform/tests/assert_destinations_group_masking.sql create mode 100644 pipeline/transform/tests/assert_destinations_no_derived_remainder.sql diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index 1aa86c1..89f2301 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -180,6 +180,74 @@ models: - name: year tests: [not_null] + - name: fact_ks4_destinations + description: > + KS4 leavers destinations — one row per URN, year, pupil group and + destination measure. status distinguishes a figure DfE withheld + ('suppressed') from one that does not apply ('not_applicable'); pupils is + null for both, so status is the only thing that tells them apart. R3 + masking is applied here: a category withheld for disadvantaged pupils is + withheld for the other-pupils group too. + columns: + - name: urn + tests: [not_null] + - name: year + tests: [not_null] + - name: pupil_group + tests: + - not_null + - accepted_values: + values: ['all', 'disadvantaged', 'other'] + - name: destination_measure + tests: [not_null] + - name: status + tests: + - not_null + - accepted_values: + values: ['published', 'suppressed', 'not_applicable'] + + - name: fact_ks5_destinations + description: > + 16-18 study leavers destinations — same grain and same suppression + semantics as fact_ks4_destinations. Only institutions with post-16 + provision appear. + columns: + - name: urn + tests: [not_null] + - name: year + tests: [not_null] + - name: pupil_group + tests: + - not_null + - accepted_values: + values: ['all', 'disadvantaged', 'other'] + - name: destination_measure + tests: [not_null] + - name: status + tests: + - not_null + - accepted_values: + values: ['published', 'suppressed', 'not_applicable'] + + - name: fact_destination_national + description: > + England destination measures by phase and pupil group — the national + reference the school sections compare against. Repointed with the cohort + switch so a school's disadvantaged pupils are measured against national + disadvantaged pupils, not against the national average. + columns: + - name: phase + tests: + - not_null + - accepted_values: + values: ['ks4', 'ks5'] + - name: year + tests: [not_null] + - name: pupil_group + tests: [not_null] + - name: destination_measure + tests: [not_null] + - name: fact_ks2_national_averages description: Official DfE KS2 national headline averages — one row per academic year columns: diff --git a/pipeline/transform/models/marts/fact_destination_national.sql b/pipeline/transform/models/marts/fact_destination_national.sql new file mode 100644 index 0000000..811078f --- /dev/null +++ b/pipeline/transform/models/marts/fact_destination_national.sql @@ -0,0 +1,40 @@ +{{ config(materialized='table') }} + +-- Mart: England destination measures by pupil group — the page's national +-- reference. +-- +-- Kept separate from the school facts so the section's England bar can repoint +-- with the cohort switch. Comparing a school's disadvantaged pupils against +-- the national all-pupils figure would flatter or damn the school for its +-- intake rather than its work, which is the failure this whole feature is +-- meant to avoid. +-- +-- The tap pins establishment type to Total within the state-funded mainstream +-- group for these rows, so the parts reconcile: for 2022/23 KS4, 151,912 +-- disadvantaged + 441,823 other = 593,735 total. + +select + 'ks4' as phase, + year, + pupil_group, + destination_measure, + cohort_pupils, + pupils, + percentage, + status +from {{ ref('stg_ees_ks4_destinations') }} +where urn is null + +union all + +select + 'ks5' as phase, + year, + pupil_group, + destination_measure, + cohort_pupils, + pupils, + percentage, + status +from {{ ref('stg_ees_ks5_destinations') }} +where urn is null diff --git a/pipeline/transform/models/marts/fact_ks4_destinations.sql b/pipeline/transform/models/marts/fact_ks4_destinations.sql new file mode 100644 index 0000000..cf03b7c --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks4_destinations.sql @@ -0,0 +1,50 @@ +{{ config(materialized='table') }} + +-- Mart: KS4 leavers destinations — one row per URN x year x pupil group x +-- destination measure. +-- +-- Long format, unlike the wide fact_ks4_performance next door. pupil_group is +-- a real third dimension, so going wide would need three sets of every column, +-- and the disclosure tests are far easier to write over rows than columns. +-- +-- R3 IS APPLIED HERE rather than in the API. Disadvantaged and other-pupils +-- partition the whole, and the all-pupils figure is published — so publishing +-- both halves recovers the suppressed one by subtraction. Where a category is +-- withheld for disadvantaged pupils it is therefore withheld for the other +-- group too, and the all-pupils view keeps it. DfE already does this in 493 of +-- 498 cases; this closes the remaining 5 so that no consumer, now or later, +-- can reach an unmasked combination. +-- +-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md. + +with staged as ( + select s.* + from {{ ref('stg_ees_ks4_destinations') }} s + inner join {{ ref('dim_school') }} d on d.urn = s.urn +), + +-- Categories withheld for disadvantaged pupils at this school and year. +masked as ( + select distinct urn, year, destination_measure + from staged + where pupil_group = 'disadvantaged' + and status = 'suppressed' +) + +select + s.urn, + s.year, + s.pupil_group, + s.destination_measure, + s.cohort_pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.pupils end as pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.percentage end as percentage, + case when m.urn is not null and s.pupil_group = 'other' + then 'suppressed' else s.status end as status +from staged s +left join masked m + on m.urn = s.urn + and m.year = s.year + and m.destination_measure = s.destination_measure diff --git a/pipeline/transform/models/marts/fact_ks5_destinations.sql b/pipeline/transform/models/marts/fact_ks5_destinations.sql new file mode 100644 index 0000000..59cbef7 --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks5_destinations.sql @@ -0,0 +1,50 @@ +{{ config(materialized='table') }} + +-- Mart: 16-18 study leavers destinations — one row per URN x year x pupil group x +-- destination measure. +-- +-- Long format, unlike the wide fact_ks4_performance next door. pupil_group is +-- a real third dimension, so going wide would need three sets of every column, +-- and the disclosure tests are far easier to write over rows than columns. +-- +-- R3 IS APPLIED HERE rather than in the API. Disadvantaged and other-pupils +-- partition the whole, and the all-pupils figure is published — so publishing +-- both halves recovers the suppressed one by subtraction. Where a category is +-- withheld for disadvantaged pupils it is therefore withheld for the other +-- group too, and the all-pupils view keeps it. DfE already does this in 493 of +-- 498 cases; this closes the remaining 5 so that no consumer, now or later, +-- can reach an unmasked combination. +-- +-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md. + +with staged as ( + select s.* + from {{ ref('stg_ees_ks5_destinations') }} s + inner join {{ ref('dim_school') }} d on d.urn = s.urn +), + +-- Categories withheld for disadvantaged pupils at this school and year. +masked as ( + select distinct urn, year, destination_measure + from staged + where pupil_group = 'disadvantaged' + and status = 'suppressed' +) + +select + s.urn, + s.year, + s.pupil_group, + s.destination_measure, + s.cohort_pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.pupils end as pupils, + case when m.urn is not null and s.pupil_group = 'other' + then null else s.percentage end as percentage, + case when m.urn is not null and s.pupil_group = 'other' + then 'suppressed' else s.status end as status +from staged s +left join masked m + on m.urn = s.urn + and m.year = s.year + and m.destination_measure = s.destination_measure diff --git a/pipeline/transform/tests/assert_destination_status_null_agreement.sql b/pipeline/transform/tests/assert_destination_status_null_agreement.sql new file mode 100644 index 0000000..ec5ffda --- /dev/null +++ b/pipeline/transform/tests/assert_destination_status_null_agreement.sql @@ -0,0 +1,15 @@ +-- pupils must be null wherever status is not 'published', and never null where +-- it is. This is what stops a later coalesce, or a wide-format refactor, +-- turning a withheld figure into a zero — which would read on the page as +-- "no pupils went here" rather than "we are not told". + +select + urn, + year, + pupil_group, + destination_measure, + status, + pupils +from {{ ref('fact_ks4_destinations') }} +where (status <> 'published' and pupils is not null) + or (status = 'published' and pupils is null) diff --git a/pipeline/transform/tests/assert_destinations_group_masking.sql b/pipeline/transform/tests/assert_destinations_group_masking.sql new file mode 100644 index 0000000..586c4ed --- /dev/null +++ b/pipeline/transform/tests/assert_destinations_group_masking.sql @@ -0,0 +1,20 @@ +-- R3 GUARD. Fails if a category is suppressed for disadvantaged pupils but +-- still published for the other-pupils group. +-- +-- The two groups partition the cohort and the all-pupils figure is published, +-- so publishing both halves recovers the withheld one. fact_ks4_destinations +-- masks the other group to prevent it; this asserts the masking actually held. + +select + d.urn, + d.year, + d.destination_measure +from {{ ref('fact_ks4_destinations') }} d +inner join {{ ref('fact_ks4_destinations') }} o + on o.urn = d.urn + and o.year = d.year + and o.destination_measure = d.destination_measure + and o.pupil_group = 'other' +where d.pupil_group = 'disadvantaged' + and d.status = 'suppressed' + and o.status = 'published' diff --git a/pipeline/transform/tests/assert_destinations_no_derived_remainder.sql b/pipeline/transform/tests/assert_destinations_no_derived_remainder.sql new file mode 100644 index 0000000..c8f53f7 --- /dev/null +++ b/pipeline/transform/tests/assert_destinations_no_derived_remainder.sql @@ -0,0 +1,24 @@ +{{ config(severity='warn') }} + +-- R1 TRIPWIRE. Flags a school/year/pupil group that has exactly one suppressed +-- category while its cohort total is published — the combination that lets the +-- withheld figure be recovered by subtracting the published categories. +-- +-- This is a WARN, not an error, and the distinction matters. The mart is not +-- wrong: DfE publishes exactly this, and the mart's job is to carry the source +-- faithfully. What must refuse to close the gap is everything downstream — the +-- API serialiser and lib/destinations.ts, which have their own tests. This +-- query exists so the condition stays visible and counted, and so anyone +-- adding a consumer later has to look at it rather than discover it. +-- +-- Expect a non-trivial count: measured at 22% of mainstream secondaries. + +select + urn, + year, + pupil_group, + count(*) filter (where status = 'suppressed') as suppressed_categories +from {{ ref('fact_ks4_destinations') }} +where destination_measure not like 'agg\_%' escape '\' +group by urn, year, pupil_group +having count(*) filter (where status = 'suppressed') = 1