feat(destinations): say what happened to a school's leavers, without republishing what DfE withheld #137

Merged
tudor merged 16 commits from feat/ks4-destinations into main 2026-08-30 20:49:15 +00:00
7 changed files with 267 additions and 0 deletions
Showing only changes of commit 5e5b61987a - Show all commits

No files matched your search

@@ -180,6 +180,74 @@ models:
- name: year
tests: [not_null]
- name: fact_ks4_destinations
description: >
KS4 leavers destinations — one row per URN, year, pupil group and
destination measure. status distinguishes a figure DfE withheld
('suppressed') from one that does not apply ('not_applicable'); pupils is
null for both, so status is the only thing that tells them apart. R3
masking is applied here: a category withheld for disadvantaged pupils is
withheld for the other-pupils group too.
columns:
- name: urn
tests: [not_null]
- name: year
tests: [not_null]
- name: pupil_group
tests:
- not_null
- accepted_values:
values: ['all', 'disadvantaged', 'other']
- name: destination_measure
tests: [not_null]
- name: status
tests:
- not_null
- accepted_values:
values: ['published', 'suppressed', 'not_applicable']
- name: fact_ks5_destinations
description: >
16-18 study leavers destinations — same grain and same suppression
semantics as fact_ks4_destinations. Only institutions with post-16
provision appear.
columns:
- name: urn
tests: [not_null]
- name: year
tests: [not_null]
- name: pupil_group
tests:
- not_null
- accepted_values:
values: ['all', 'disadvantaged', 'other']
- name: destination_measure
tests: [not_null]
- name: status
tests:
- not_null
- accepted_values:
values: ['published', 'suppressed', 'not_applicable']
- name: fact_destination_national
description: >
England destination measures by phase and pupil group — the national
reference the school sections compare against. Repointed with the cohort
switch so a school's disadvantaged pupils are measured against national
disadvantaged pupils, not against the national average.
columns:
- name: phase
tests:
- not_null
- accepted_values:
values: ['ks4', 'ks5']
- name: year
tests: [not_null]
- name: pupil_group
tests: [not_null]
- name: destination_measure
tests: [not_null]
- name: fact_ks2_national_averages
description: Official DfE KS2 national headline averages — one row per academic year
columns:
@@ -0,0 +1,40 @@
{{ config(materialized='table') }}
-- Mart: England destination measures by pupil group — the page's national
-- reference.
--
-- Kept separate from the school facts so the section's England bar can repoint
-- with the cohort switch. Comparing a school's disadvantaged pupils against
-- the national all-pupils figure would flatter or damn the school for its
-- intake rather than its work, which is the failure this whole feature is
-- meant to avoid.
--
-- The tap pins establishment type to Total within the state-funded mainstream
-- group for these rows, so the parts reconcile: for 2022/23 KS4, 151,912
-- disadvantaged + 441,823 other = 593,735 total.
select
'ks4' as phase,
year,
pupil_group,
destination_measure,
cohort_pupils,
pupils,
percentage,
status
from {{ ref('stg_ees_ks4_destinations') }}
where urn is null
union all
select
'ks5' as phase,
year,
pupil_group,
destination_measure,
cohort_pupils,
pupils,
percentage,
status
from {{ ref('stg_ees_ks5_destinations') }}
where urn is null
@@ -0,0 +1,50 @@
{{ config(materialized='table') }}
-- Mart: KS4 leavers destinations — one row per URN x year x pupil group x
-- destination measure.
--
-- Long format, unlike the wide fact_ks4_performance next door. pupil_group is
-- a real third dimension, so going wide would need three sets of every column,
-- and the disclosure tests are far easier to write over rows than columns.
--
-- R3 IS APPLIED HERE rather than in the API. Disadvantaged and other-pupils
-- partition the whole, and the all-pupils figure is published — so publishing
-- both halves recovers the suppressed one by subtraction. Where a category is
-- withheld for disadvantaged pupils it is therefore withheld for the other
-- group too, and the all-pupils view keeps it. DfE already does this in 493 of
-- 498 cases; this closes the remaining 5 so that no consumer, now or later,
-- can reach an unmasked combination.
--
-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md.
with staged as (
select s.*
from {{ ref('stg_ees_ks4_destinations') }} s
inner join {{ ref('dim_school') }} d on d.urn = s.urn
),
-- Categories withheld for disadvantaged pupils at this school and year.
masked as (
select distinct urn, year, destination_measure
from staged
where pupil_group = 'disadvantaged'
and status = 'suppressed'
)
select
s.urn,
s.year,
s.pupil_group,
s.destination_measure,
s.cohort_pupils,
case when m.urn is not null and s.pupil_group = 'other'
then null else s.pupils end as pupils,
case when m.urn is not null and s.pupil_group = 'other'
then null else s.percentage end as percentage,
case when m.urn is not null and s.pupil_group = 'other'
then 'suppressed' else s.status end as status
from staged s
left join masked m
on m.urn = s.urn
and m.year = s.year
and m.destination_measure = s.destination_measure
@@ -0,0 +1,50 @@
{{ config(materialized='table') }}
-- Mart: 16-18 study leavers destinations — one row per URN x year x pupil group x
-- destination measure.
--
-- Long format, unlike the wide fact_ks4_performance next door. pupil_group is
-- a real third dimension, so going wide would need three sets of every column,
-- and the disclosure tests are far easier to write over rows than columns.
--
-- R3 IS APPLIED HERE rather than in the API. Disadvantaged and other-pupils
-- partition the whole, and the all-pupils figure is published — so publishing
-- both halves recovers the suppressed one by subtraction. Where a category is
-- withheld for disadvantaged pupils it is therefore withheld for the other
-- group too, and the all-pupils view keeps it. DfE already does this in 493 of
-- 498 cases; this closes the remaining 5 so that no consumer, now or later,
-- can reach an unmasked combination.
--
-- See docs/superpowers/specs/2026-08-28-destination-measures-design.md.
with staged as (
select s.*
from {{ ref('stg_ees_ks5_destinations') }} s
inner join {{ ref('dim_school') }} d on d.urn = s.urn
),
-- Categories withheld for disadvantaged pupils at this school and year.
masked as (
select distinct urn, year, destination_measure
from staged
where pupil_group = 'disadvantaged'
and status = 'suppressed'
)
select
s.urn,
s.year,
s.pupil_group,
s.destination_measure,
s.cohort_pupils,
case when m.urn is not null and s.pupil_group = 'other'
then null else s.pupils end as pupils,
case when m.urn is not null and s.pupil_group = 'other'
then null else s.percentage end as percentage,
case when m.urn is not null and s.pupil_group = 'other'
then 'suppressed' else s.status end as status
from staged s
left join masked m
on m.urn = s.urn
and m.year = s.year
and m.destination_measure = s.destination_measure
@@ -0,0 +1,15 @@
-- pupils must be null wherever status is not 'published', and never null where
-- it is. This is what stops a later coalesce, or a wide-format refactor,
-- turning a withheld figure into a zero — which would read on the page as
-- "no pupils went here" rather than "we are not told".
select
urn,
year,
pupil_group,
destination_measure,
status,
pupils
from {{ ref('fact_ks4_destinations') }}
where (status <> 'published' and pupils is not null)
or (status = 'published' and pupils is null)
@@ -0,0 +1,20 @@
-- R3 GUARD. Fails if a category is suppressed for disadvantaged pupils but
-- still published for the other-pupils group.
--
-- The two groups partition the cohort and the all-pupils figure is published,
-- so publishing both halves recovers the withheld one. fact_ks4_destinations
-- masks the other group to prevent it; this asserts the masking actually held.
select
d.urn,
d.year,
d.destination_measure
from {{ ref('fact_ks4_destinations') }} d
inner join {{ ref('fact_ks4_destinations') }} o
on o.urn = d.urn
and o.year = d.year
and o.destination_measure = d.destination_measure
and o.pupil_group = 'other'
where d.pupil_group = 'disadvantaged'
and d.status = 'suppressed'
and o.status = 'published'
@@ -0,0 +1,24 @@
{{ config(severity='warn') }}
-- R1 TRIPWIRE. Flags a school/year/pupil group that has exactly one suppressed
-- category while its cohort total is published — the combination that lets the
-- withheld figure be recovered by subtracting the published categories.
--
-- This is a WARN, not an error, and the distinction matters. The mart is not
-- wrong: DfE publishes exactly this, and the mart's job is to carry the source
-- faithfully. What must refuse to close the gap is everything downstream — the
-- API serialiser and lib/destinations.ts, which have their own tests. This
-- query exists so the condition stays visible and counted, and so anyone
-- adding a consumer later has to look at it rather than discover it.
--
-- Expect a non-trivial count: measured at 22% of mainstream secondaries.
select
urn,
year,
pupil_group,
count(*) filter (where status = 'suppressed') as suppressed_categories
from {{ ref('fact_ks4_destinations') }}
where destination_measure not like 'agg\_%' escape '\'
group by urn, year, pupil_group
having count(*) filter (where status = 'suppressed') = 1