From 519584f34ba8d56d0bad9566e3799e1b26b3de88 Mon Sep 17 00:00:00 2001 From: Tudor Date: Mon, 13 Jul 2026 23:54:20 +0100 Subject: [PATCH] feat(compare): trends explorer with England line; gap-honest axis; series regression guard Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB --- .../__tests__/lib/compareChartData.test.ts | 84 ++++++++ .../components/ComparisonChart.module.css | 6 + nextjs-app/components/ComparisonChart.tsx | 56 +++-- .../compare/TrendsExplorer.module.css | 96 +++++++++ .../components/compare/TrendsExplorer.tsx | 195 ++++++++++++++++++ nextjs-app/lib/compareChartData.ts | 102 +++++++++ 6 files changed, 523 insertions(+), 16 deletions(-) create mode 100644 nextjs-app/__tests__/lib/compareChartData.test.ts create mode 100644 nextjs-app/components/compare/TrendsExplorer.module.css create mode 100644 nextjs-app/components/compare/TrendsExplorer.tsx create mode 100644 nextjs-app/lib/compareChartData.ts diff --git a/nextjs-app/__tests__/lib/compareChartData.test.ts b/nextjs-app/__tests__/lib/compareChartData.test.ts new file mode 100644 index 0000000..09325ae --- /dev/null +++ b/nextjs-app/__tests__/lib/compareChartData.test.ts @@ -0,0 +1,84 @@ +/** + * buildCompareChart: every selected school must produce a rendered series + * (regression guard for the production bug where a third school's line + * vanished), the x-axis must include cancelled/unpublished years as real + * gaps (never compressing time), and the England overlay renders dashed + * with no gap-bridging. + */ + +import { buildCompareChart, fillAcademicYears } from '@/lib/compareChartData'; +import type { ComparisonData } from '@/lib/types'; + +function school(urn: number, years: Array<[number, number | null]>): ComparisonData { + return { + school_info: { urn, school_name: `School ${urn}` } as ComparisonData['school_info'], + yearly_data: years.map(([year, v]) => ({ year, rwm_expected_pct: v })) as ComparisonData['yearly_data'], + }; +} + +const THREE_SCHOOLS = { + '1': school(1, [[201819, 87], [202223, 87], [202425, 87]]), + '2': school(2, [[201819, 88], [202223, 88], [202425, 92]]), + '3': school(3, [[201819, 69], [202223, 62], [202425, 79]]), +}; + +const SCHOOL_LIST = [1, 2, 3].map((urn) => ({ urn, school_name: `School ${urn}` })); + +describe('fillAcademicYears', () => { + it('fills every academic year between min and max', () => { + expect(fillAcademicYears([201819, 202223])).toEqual([ + 201819, 201920, 202021, 202122, 202223, + ]); + }); +}); + +describe('buildCompareChart', () => { + it('renders one series per selected school — none silently dropped', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(chart.schoolDatasets).toHaveLength(3); + for (const ds of chart.schoolDatasets) { + expect(ds.data.some((v) => v != null)).toBe(true); + } + }); + + it('handles float years from the API (202425.0 style)', () => { + const floaty = { + '1': school(1, [[201819.0 as number, 80], [202425.0 as number, 85]]), + }; + const chart = buildCompareChart(floaty, [SCHOOL_LIST[0]], 'rwm_expected_pct'); + expect(chart.schoolDatasets[0].data.filter((v) => v != null)).toHaveLength(2); + }); + + it('includes cancelled/unpublished years as null gaps, not compressed time', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(chart.years).toContain(201920); + expect(chart.years).toContain(202122); + const idx = chart.years.indexOf(202021); + expect(chart.schoolDatasets[0].data[idx]).toBeNull(); + }); + + it('adds a dashed England overlay when national data is supplied', () => { + const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', { + 201819: 64.9, + 202122: 58.7, + 202223: 59.5, + 202425: 62.1, + }); + expect(chart.englandDataset).not.toBeNull(); + const eng = chart.englandDataset!; + expect(eng.label).toBe('England average'); + expect(eng.borderDash).toEqual([5, 4]); + expect(eng.spanGaps).toBe(false); + // England has a value for 2021/22 even though schools do not + expect(eng.data[chart.years.indexOf(202122)]).toBe(58.7); + }); + + it('flags the unpublished 2021/22 school-level year when England has data but schools do not', () => { + const withNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', { + 202122: 58.7, + }); + expect(withNational.showUnpublished202122Note).toBe(true); + const withoutNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct'); + expect(withoutNational.showUnpublished202122Note).toBe(false); + }); +}); diff --git a/nextjs-app/components/ComparisonChart.module.css b/nextjs-app/components/ComparisonChart.module.css index dbd82c6..d847f1d 100644 --- a/nextjs-app/components/ComparisonChart.module.css +++ b/nextjs-app/components/ComparisonChart.module.css @@ -65,3 +65,9 @@ min-width: 0; } } + +.chartNote { + font-size: 0.78rem; + color: var(--text-muted); + margin: 0.5rem 0 0; +} diff --git a/nextjs-app/components/ComparisonChart.tsx b/nextjs-app/components/ComparisonChart.tsx index d922991..851b03d 100644 --- a/nextjs-app/components/ComparisonChart.tsx +++ b/nextjs-app/components/ComparisonChart.tsx @@ -15,6 +15,7 @@ import { useEffect, useState } from 'react'; import { Line } from 'react-chartjs-2'; import { ChartOptions, ChartDataset, PointStyle } from 'chart.js'; import '@/lib/chartSetup'; +import { buildCompareChart } from '@/lib/compareChartData'; import type { ComparisonData } from '@/lib/types'; import { CHART_COLORS, @@ -34,13 +35,16 @@ interface ComparisonChartProps { schools: Array<{ urn: number; school_name: string }>; metric: string; metricLabel: string; + /** Official England figure per academic year for this metric — renders a + * dashed grey reference line when provided. */ + nationalByYear?: Record; } // One shape per basket slot (MAX_SCHOOLS = 5) — secondary encoding so // converging lines stay tellable apart without relying on hue alone. const POINT_STYLES: PointStyle[] = ['circle', 'triangle', 'rect', 'rectRot', 'star']; -export function ComparisonChart({ comparisonData, schools, metric, metricLabel }: ComparisonChartProps) { +export function ComparisonChart({ comparisonData, schools, metric, metricLabel, nationalByYear }: ComparisonChartProps) { const isMobile = useIsMobile(); const [focusedUrn, setFocusedUrn] = useState(null); @@ -54,34 +58,48 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel } return
No data available
; } - // Union of years across all schools — coverage differs between them. - const years = [ - ...new Set(schools.flatMap((s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => d.year) ?? [])), - ].sort((a, b) => a - b); + // Pure, tested series construction: union of years with cancelled / + // unpublished years kept as real gaps, plus the England overlay. + const built = buildCompareChart(comparisonData, schools, metric, nationalByYear); + const { years } = built; - const datasets: ChartDataset<'line'>[] = schools.map((school, index) => { - const data = comparisonData[String(school.urn)]; - const color = CHART_COLORS[index % CHART_COLORS.length]; + const datasets: ChartDataset<'line'>[] = built.schoolDatasets.map((series) => { + const school = schools[series.schoolIndex]; + const color = CHART_COLORS[series.schoolIndex % CHART_COLORS.length]; const dimmed = focusedUrn !== null && focusedUrn !== school.urn; return { - label: school.school_name, - data: years.map((year) => { - const yearData = data?.yearly_data.find((d) => d.year === year); - if (!yearData) return null; - return yearData[metric as keyof typeof yearData] as number | null; - }), + label: series.label, + data: series.data, borderColor: dimmed ? rgbToRgba(color, 0.2) : color, backgroundColor: dimmed ? 'transparent' : rgbToRgba(color, 0.1), borderWidth: focusedUrn === school.urn ? 3 : dimmed ? 1.5 : 2, - pointStyle: POINT_STYLES[index % POINT_STYLES.length], + pointStyle: POINT_STYLES[series.schoolIndex % POINT_STYLES.length], pointRadius: dimmed ? 2 : isMobile ? 3 : 4, pointHoverRadius: isMobile ? 5 : 6, tension: 0.3, - spanGaps: true, + // Never bridge missing years — gaps are information (COVID + // cancellations, unpublished 2021/22, schools that opened later). + spanGaps: false, }; }); + if (built.englandDataset) { + datasets.push({ + label: built.englandDataset.label, + data: built.englandDataset.data, + borderColor: 'rgba(109, 104, 95, 0.9)', + backgroundColor: 'transparent', + borderWidth: 1.5, + borderDash: built.englandDataset.borderDash, + pointStyle: 'line', + pointRadius: 0, + pointHoverRadius: 4, + tension: 0, + spanGaps: false, + }); + } + const chartData = { labels: years.map(formatAcademicYear), datasets, @@ -222,6 +240,12 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
+ {built.showUnpublished202122Note && ( +

+ No national tests were held in 2019/20 and 2020/21 (COVID), and DfE didn't publish + school-level figures for 2021/22 — the England average is shown for that year. +

+ )} ); } diff --git a/nextjs-app/components/compare/TrendsExplorer.module.css b/nextjs-app/components/compare/TrendsExplorer.module.css new file mode 100644 index 0000000..d6ddd4d --- /dev/null +++ b/nextjs-app/components/compare/TrendsExplorer.module.css @@ -0,0 +1,96 @@ +.explore { + margin-top: 1rem; +} + +.explore summary { + cursor: pointer; + font-weight: 600; + color: var(--accent-coral-dark); + padding: 0.85rem 1.1rem; + background: var(--bg-card); + border: 1px solid var(--border-light); + border-radius: 8px; +} + +.explore[open] summary { + border-radius: 8px 8px 0 0; +} + +.inner { + border: 1px solid var(--border-light); + border-top: none; + border-radius: 0 0 8px 8px; + background: var(--bg-card); + padding: 1.25rem 1.5rem; +} + +.picker { + display: flex; + align-items: center; + gap: 0.6rem; + margin-bottom: 1rem; + flex-wrap: wrap; +} + +.picker label { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-secondary); +} + +.picker select { + font-family: inherit; + font-size: 0.9rem; + padding: 0.4rem 0.6rem; + border-radius: 8px; + border: 1px solid var(--border-light); + background: var(--bg-card); + color: var(--text-primary); + max-width: 100%; +} + +.desc { + font-size: 0.78rem; + color: var(--text-muted); +} + +.progressNote { + font-size: 0.8rem; + color: var(--text-muted); + margin: 0 0 1rem; +} + +.chartBox { + min-height: 320px; +} + +.tableWrapper { + overflow-x: auto; + margin-top: 1.5rem; +} + +.table { + width: 100%; + border-collapse: collapse; + font-size: 0.9rem; +} + +.table th, +.table td { + text-align: left; + padding: 0.6rem 0.75rem; + border-bottom: 1px solid var(--border-light); +} + +.table th { + background: var(--bg-secondary); + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.03em; + color: var(--text-secondary); +} + +.yearCell { + font-weight: 600; + white-space: nowrap; +} diff --git a/nextjs-app/components/compare/TrendsExplorer.tsx b/nextjs-app/components/compare/TrendsExplorer.tsx new file mode 100644 index 0000000..c4ffd64 --- /dev/null +++ b/nextjs-app/components/compare/TrendsExplorer.tsx @@ -0,0 +1,195 @@ +/** + * Explore trends — the full grouped metric catalogue (nothing from the old + * compare page is lost; spec §4's tier 3) driving the year-by-year chart + * with its England reference line, plus the year-by-year table. Progress + * metrics carry CI-based bands for the years DfE published them. + */ + +'use client'; + +import { useState } from 'react'; +import dynamic from 'next/dynamic'; + +import { progressBand } from '@/lib/compareLogic'; +import type { ComparisonData, MetricDefinition, NationalAverages, School } from '@/lib/types'; +import { formatAcademicYear, formatMetricValue, metricKind } from '@/lib/utils'; +import { track } from '@/lib/analytics'; +import { Chip, Section, sectionStyles as s } from './sectionShared'; +import styles from './TrendsExplorer.module.css'; + +const ComparisonChart = dynamic( + () => import('../ComparisonChart').then((m) => m.ComparisonChart), + { ssr: false }, +); + +const PRIMARY_OPTGROUPS: { label: string; category: string }[] = [ + { label: 'Expected Standard', category: 'expected' }, + { label: 'Higher Standard', category: 'higher' }, + { label: 'Progress Scores', category: 'progress' }, + { label: 'Average Scores', category: 'average' }, + { label: 'Gender Performance', category: 'gender' }, + { label: 'Equity (Disadvantaged)', category: 'equity' }, + { label: 'School Context', category: 'context' }, + { label: 'Absence', category: 'absence' }, + { label: '3-Year Trends', category: 'trends' }, +]; + +const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [ + { label: 'GCSE Performance', category: 'gcse' }, +]; + +export const PRIMARY_CATEGORIES = PRIMARY_OPTGROUPS.map((g) => g.category); +export const SECONDARY_CATEGORIES = SECONDARY_OPTGROUPS.map((g) => g.category); + +const PROGRESS_CI: Record = { + reading_progress: ['reading_progress_lower_ci', 'reading_progress_upper_ci'], + writing_progress: ['writing_progress_lower_ci', 'writing_progress_upper_ci'], + maths_progress: ['maths_progress_lower_ci', 'maths_progress_upper_ci'], +}; + +const BAND_LABEL = { above: 'Above average', average: 'Average', below: 'Below average' } as const; + +export function TrendsExplorer({ + schools, + data, + metrics, + initialMetric, + isPrimaryPhase, + nationalAverages, +}: { + schools: School[]; + data: Record; + metrics: MetricDefinition[]; + initialMetric: string; + isPrimaryPhase: boolean; + nationalAverages?: NationalAverages; +}) { + const [metric, setMetric] = useState(initialMetric); + + const allowedCategories = isPrimaryPhase ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES; + const optgroups = isPrimaryPhase ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS; + const filteredMetrics = metrics.filter((m) => allowedCategories.includes(m.category)); + const metricDef = metrics.find((m) => m.key === metric); + const metricLabel = metricDef?.label || metric; + + const nationalByYear: Record = {}; + for (const entry of nationalAverages?.by_year ?? []) { + const block = isPrimaryPhase ? entry.primary : entry.secondary; + nationalByYear[entry.year] = block?.[metric] ?? null; + } + + const years = [ + ...new Set( + schools.flatMap( + (school) => data[String(school.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [], + ), + ), + ].sort((a, b) => a - b); + + const handleMetricChange = (next: string) => { + track('compare_metric_changed', { metric: next, phase: isPrimaryPhase ? 'primary' : 'secondary' }); + setMetric(next); + }; + + const ciKeys = PROGRESS_CI[metric]; + + return ( +
+
+ Year-by-year trends +
+
+ + + {metricDef?.description && {metricDef.description}} +
+ + {metric.includes('progress') && ( +

+ Progress scores measure pupils' progress from KS1 to KS2. A score of 0 equals the + national average. DfE stopped publishing KS2 progress after 2022/23 (no KS1 baseline); + bands use DfE's confidence intervals, not the raw score alone. +

+ )} + +
+ +
+ + {years.length > 0 && ( +
+ + + + + {schools.map((school) => ( + + ))} + + + + {years.map((year) => ( + + + {schools.map((school) => { + const row = data[String(school.urn)]?.yearly_data.find( + (d) => Math.trunc(d.year) === year, + ) as (Record & { year: number }) | undefined; + const value = row?.[metric]; + if (typeof value !== 'number') return ; + const band = ciKeys + ? progressBand( + value, + (row?.[ciKeys[0]] as number | null) ?? null, + (row?.[ciKeys[1]] as number | null) ?? null, + ) + : null; + return ( + + ); + })} + + ))} + +
Year{school.school_name}
{formatAcademicYear(year)}– + {formatMetricValue(value, metricKind(metric))}{' '} + {band && ( + + {BAND_LABEL[band]} + + )} +
+
+ )} +
+
+
+ ); +} diff --git a/nextjs-app/lib/compareChartData.ts b/nextjs-app/lib/compareChartData.ts new file mode 100644 index 0000000..728fd7a --- /dev/null +++ b/nextjs-app/lib/compareChartData.ts @@ -0,0 +1,102 @@ +/** + * Pure series-building for the comparison trend chart, extracted from + * ComparisonChart so it is unit-testable without a canvas. + * + * Chart truthfulness rules (spec §8.1): every academic year between the + * first and last data point appears on the axis — cancelled test years + * (2019/20, 2020/21) and the unpublished 2021/22 school-level year render + * as real gaps, never as compressed time; school lines never bridge gaps. + */ + +import type { ComparisonData } from './types'; + +/** 201819 → 201920 (academic-year arithmetic on YYYYYY codes). */ +function nextAcademicYear(year: number): number { + const start = Math.floor(year / 100); + const end = year % 100; + return (start + 1) * 100 + (end + 1); +} + +/** Every academic year from min(years) to max(years), inclusive. */ +export function fillAcademicYears(years: number[]): number[] { + if (years.length === 0) return []; + const ints = [...new Set(years.map((y) => Math.trunc(y)))].sort((a, b) => a - b); + const out: number[] = []; + let y = ints[0]; + const last = ints[ints.length - 1]; + while (y <= last && out.length < 50) { + out.push(y); + y = nextAcademicYear(y); + } + return out; +} + +export interface CompareChartSeries { + label: string; + data: Array; + /** Index into CHART_COLORS / point styles. */ + schoolIndex: number; + spanGaps: false; +} + +export interface EnglandSeries { + label: 'England average'; + data: Array; + borderDash: [number, number]; + spanGaps: false; +} + +export interface CompareChart { + years: number[]; + schoolDatasets: CompareChartSeries[]; + englandDataset: EnglandSeries | null; + /** True when England published a 2021/22 figure but no school has one — + * the UI shows: "DfE didn't publish school-level figures for 2021/22". */ + showUnpublished202122Note: boolean; +} + +export function buildCompareChart( + comparisonData: Record, + schools: Array<{ urn: number; school_name: string }>, + metric: string, + nationalByYear?: Record, +): CompareChart { + const rawYears = schools.flatMap( + (s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [], + ); + const years = fillAcademicYears(rawYears); + + const schoolDatasets: CompareChartSeries[] = schools.map((school, schoolIndex) => { + const rows = comparisonData[String(school.urn)]?.yearly_data ?? []; + const byYear = new Map>(); + for (const row of rows) byYear.set(Math.trunc(row.year), row as unknown as Record); + return { + label: school.school_name, + data: years.map((year) => { + const v = byYear.get(year)?.[metric]; + return typeof v === 'number' && !Number.isNaN(v) ? v : null; + }), + schoolIndex, + spanGaps: false, + }; + }); + + let englandDataset: EnglandSeries | null = null; + if (nationalByYear) { + const data = years.map((year) => { + const v = nationalByYear[year]; + return typeof v === 'number' && !Number.isNaN(v) ? v : null; + }); + if (data.some((v) => v != null)) { + englandDataset = { label: 'England average', data, borderDash: [5, 4], spanGaps: false }; + } + } + + const idx202122 = years.indexOf(202122); + const showUnpublished202122Note = + idx202122 >= 0 && + englandDataset?.data[idx202122] != null && + schoolDatasets.every((ds) => ds.data[idx202122] == null); + + return { years, schoolDatasets, englandDataset, showUnpublished202122Note }; +}