feat(compare): trends explorer with England line; gap-honest axis; series regression guard

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB

nextjs-app/__tests__/lib/compareChartData.test.ts

@@ -0,0 +1,84 @@
+/**
+ * buildCompareChart: every selected school must produce a rendered series
+ * (regression guard for the production bug where a third school's line
+ * vanished), the x-axis must include cancelled/unpublished years as real
+ * gaps (never compressing time), and the England overlay renders dashed
+ * with no gap-bridging.
+ */
+
+import { buildCompareChart, fillAcademicYears } from '@/lib/compareChartData';
+import type { ComparisonData } from '@/lib/types';
+
+function school(urn: number, years: Array<[number, number | null]>): ComparisonData {
+  return {
+    school_info: { urn, school_name: `School ${urn}` } as ComparisonData['school_info'],
+    yearly_data: years.map(([year, v]) => ({ year, rwm_expected_pct: v })) as ComparisonData['yearly_data'],
+  };
+}
+
+const THREE_SCHOOLS = {
+  '1': school(1, [[201819, 87], [202223, 87], [202425, 87]]),
+  '2': school(2, [[201819, 88], [202223, 88], [202425, 92]]),
+  '3': school(3, [[201819, 69], [202223, 62], [202425, 79]]),
+};
+
+const SCHOOL_LIST = [1, 2, 3].map((urn) => ({ urn, school_name: `School ${urn}` }));
+
+describe('fillAcademicYears', () => {
+  it('fills every academic year between min and max', () => {
+    expect(fillAcademicYears([201819, 202223])).toEqual([
+      201819, 201920, 202021, 202122, 202223,
+    ]);
+  });
+});
+
+describe('buildCompareChart', () => {
+  it('renders one series per selected school — none silently dropped', () => {
+    const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct');
+    expect(chart.schoolDatasets).toHaveLength(3);
+    for (const ds of chart.schoolDatasets) {
+      expect(ds.data.some((v) => v != null)).toBe(true);
+    }
+  });
+
+  it('handles float years from the API (202425.0 style)', () => {
+    const floaty = {
+      '1': school(1, [[201819.0 as number, 80], [202425.0 as number, 85]]),
+    };
+    const chart = buildCompareChart(floaty, [SCHOOL_LIST[0]], 'rwm_expected_pct');
+    expect(chart.schoolDatasets[0].data.filter((v) => v != null)).toHaveLength(2);
+  });
+
+  it('includes cancelled/unpublished years as null gaps, not compressed time', () => {
+    const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct');
+    expect(chart.years).toContain(201920);
+    expect(chart.years).toContain(202122);
+    const idx = chart.years.indexOf(202021);
+    expect(chart.schoolDatasets[0].data[idx]).toBeNull();
+  });
+
+  it('adds a dashed England overlay when national data is supplied', () => {
+    const chart = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', {
+      201819: 64.9,
+      202122: 58.7,
+      202223: 59.5,
+      202425: 62.1,
+    });
+    expect(chart.englandDataset).not.toBeNull();
+    const eng = chart.englandDataset!;
+    expect(eng.label).toBe('England average');
+    expect(eng.borderDash).toEqual([5, 4]);
+    expect(eng.spanGaps).toBe(false);
+    // England has a value for 2021/22 even though schools do not
+    expect(eng.data[chart.years.indexOf(202122)]).toBe(58.7);
+  });
+
+  it('flags the unpublished 2021/22 school-level year when England has data but schools do not', () => {
+    const withNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct', {
+      202122: 58.7,
+    });
+    expect(withNational.showUnpublished202122Note).toBe(true);
+    const withoutNational = buildCompareChart(THREE_SCHOOLS, SCHOOL_LIST, 'rwm_expected_pct');
+    expect(withoutNational.showUnpublished202122Note).toBe(false);
+  });
+});

nextjs-app/components/ComparisonChart.module.css

@@ -65,3 +65,9 @@
         min-width: 0;
     }
 }
+
+.chartNote {
+  font-size: 0.78rem;
+  color: var(--text-muted);
+  margin: 0.5rem 0 0;
+}

nextjs-app/components/ComparisonChart.tsx

@@ -15,6 +15,7 @@ import { useEffect, useState } from 'react';
 import { Line } from 'react-chartjs-2';
 import { ChartOptions, ChartDataset, PointStyle } from 'chart.js';
 import '@/lib/chartSetup';
+import { buildCompareChart } from '@/lib/compareChartData';
 import type { ComparisonData } from '@/lib/types';
 import {
   CHART_COLORS,
@@ -34,13 +35,16 @@ interface ComparisonChartProps {
   schools: Array<{ urn: number; school_name: string }>;
   metric: string;
   metricLabel: string;
+  /** Official England figure per academic year for this metric — renders a
+   * dashed grey reference line when provided. */
+  nationalByYear?: Record<number, number | null | undefined>;
 }
 
 // One shape per basket slot (MAX_SCHOOLS = 5) — secondary encoding so
 // converging lines stay tellable apart without relying on hue alone.
 const POINT_STYLES: PointStyle[] = ['circle', 'triangle', 'rect', 'rectRot', 'star'];
 
-export function ComparisonChart({ comparisonData, schools, metric, metricLabel }: ComparisonChartProps) {
+export function ComparisonChart({ comparisonData, schools, metric, metricLabel, nationalByYear }: ComparisonChartProps) {
   const isMobile = useIsMobile();
   const [focusedUrn, setFocusedUrn] = useState<number | null>(null);
 
@@ -54,34 +58,48 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
     return <div>No data available</div>;
   }
 
-  // Union of years across all schools — coverage differs between them.
+  // Pure, tested series construction: union of years with cancelled /
-  const years = [
+  // unpublished years kept as real gaps, plus the England overlay.
-    ...new Set(schools.flatMap((s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => d.year) ?? [])),
+  const built = buildCompareChart(comparisonData, schools, metric, nationalByYear);
-  ].sort((a, b) => a - b);
+  const { years } = built;
 
-  const datasets: ChartDataset<'line'>[] = schools.map((school, index) => {
+  const datasets: ChartDataset<'line'>[] = built.schoolDatasets.map((series) => {
-    const data = comparisonData[String(school.urn)];
+    const school = schools[series.schoolIndex];
-    const color = CHART_COLORS[index % CHART_COLORS.length];
+    const color = CHART_COLORS[series.schoolIndex % CHART_COLORS.length];
     const dimmed = focusedUrn !== null && focusedUrn !== school.urn;
 
     return {
-      label: school.school_name,
+      label: series.label,
-      data: years.map((year) => {
+      data: series.data,
-        const yearData = data?.yearly_data.find((d) => d.year === year);
-        if (!yearData) return null;
-        return yearData[metric as keyof typeof yearData] as number | null;
-      }),
       borderColor: dimmed ? rgbToRgba(color, 0.2) : color,
       backgroundColor: dimmed ? 'transparent' : rgbToRgba(color, 0.1),
       borderWidth: focusedUrn === school.urn ? 3 : dimmed ? 1.5 : 2,
-      pointStyle: POINT_STYLES[index % POINT_STYLES.length],
+      pointStyle: POINT_STYLES[series.schoolIndex % POINT_STYLES.length],
       pointRadius: dimmed ? 2 : isMobile ? 3 : 4,
       pointHoverRadius: isMobile ? 5 : 6,
       tension: 0.3,
-      spanGaps: true,
+      // Never bridge missing years — gaps are information (COVID
+      // cancellations, unpublished 2021/22, schools that opened later).
+      spanGaps: false,
     };
   });
 
+  if (built.englandDataset) {
+    datasets.push({
+      label: built.englandDataset.label,
+      data: built.englandDataset.data,
+      borderColor: 'rgba(109, 104, 95, 0.9)',
+      backgroundColor: 'transparent',
+      borderWidth: 1.5,
+      borderDash: built.englandDataset.borderDash,
+      pointStyle: 'line',
+      pointRadius: 0,
+      pointHoverRadius: 4,
+      tension: 0,
+      spanGaps: false,
+    });
+  }
+
   const chartData = {
     labels: years.map(formatAcademicYear),
     datasets,
@@ -222,6 +240,12 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
       <div className={styles.canvasBox}>
         <Line data={chartData} options={options} aria-label={`${metricLabel} comparison chart`} />
       </div>
+      {built.showUnpublished202122Note && (
+        <p className={styles.chartNote}>
+          No national tests were held in 2019/20 and 2020/21 (COVID), and DfE didn&apos;t publish
+          school-level figures for 2021/22 — the England average is shown for that year.
+        </p>
+      )}
     </div>
   );
 }

nextjs-app/components/compare/TrendsExplorer.module.css

@@ -0,0 +1,96 @@
+.explore {
+  margin-top: 1rem;
+}
+
+.explore summary {
+  cursor: pointer;
+  font-weight: 600;
+  color: var(--accent-coral-dark);
+  padding: 0.85rem 1.1rem;
+  background: var(--bg-card);
+  border: 1px solid var(--border-light);
+  border-radius: 8px;
+}
+
+.explore[open] summary {
+  border-radius: 8px 8px 0 0;
+}
+
+.inner {
+  border: 1px solid var(--border-light);
+  border-top: none;
+  border-radius: 0 0 8px 8px;
+  background: var(--bg-card);
+  padding: 1.25rem 1.5rem;
+}
+
+.picker {
+  display: flex;
+  align-items: center;
+  gap: 0.6rem;
+  margin-bottom: 1rem;
+  flex-wrap: wrap;
+}
+
+.picker label {
+  font-size: 0.85rem;
+  font-weight: 600;
+  color: var(--text-secondary);
+}
+
+.picker select {
+  font-family: inherit;
+  font-size: 0.9rem;
+  padding: 0.4rem 0.6rem;
+  border-radius: 8px;
+  border: 1px solid var(--border-light);
+  background: var(--bg-card);
+  color: var(--text-primary);
+  max-width: 100%;
+}
+
+.desc {
+  font-size: 0.78rem;
+  color: var(--text-muted);
+}
+
+.progressNote {
+  font-size: 0.8rem;
+  color: var(--text-muted);
+  margin: 0 0 1rem;
+}
+
+.chartBox {
+  min-height: 320px;
+}
+
+.tableWrapper {
+  overflow-x: auto;
+  margin-top: 1.5rem;
+}
+
+.table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 0.9rem;
+}
+
+.table th,
+.table td {
+  text-align: left;
+  padding: 0.6rem 0.75rem;
+  border-bottom: 1px solid var(--border-light);
+}
+
+.table th {
+  background: var(--bg-secondary);
+  font-size: 0.8rem;
+  text-transform: uppercase;
+  letter-spacing: 0.03em;
+  color: var(--text-secondary);
+}
+
+.yearCell {
+  font-weight: 600;
+  white-space: nowrap;
+}

nextjs-app/components/compare/TrendsExplorer.tsx

@@ -0,0 +1,195 @@
+/**
+ * Explore trends — the full grouped metric catalogue (nothing from the old
+ * compare page is lost; spec §4's tier 3) driving the year-by-year chart
+ * with its England reference line, plus the year-by-year table. Progress
+ * metrics carry CI-based bands for the years DfE published them.
+ */
+
+'use client';
+
+import { useState } from 'react';
+import dynamic from 'next/dynamic';
+
+import { progressBand } from '@/lib/compareLogic';
+import type { ComparisonData, MetricDefinition, NationalAverages, School } from '@/lib/types';
+import { formatAcademicYear, formatMetricValue, metricKind } from '@/lib/utils';
+import { track } from '@/lib/analytics';
+import { Chip, Section, sectionStyles as s } from './sectionShared';
+import styles from './TrendsExplorer.module.css';
+
+const ComparisonChart = dynamic(
+  () => import('../ComparisonChart').then((m) => m.ComparisonChart),
+  { ssr: false },
+);
+
+const PRIMARY_OPTGROUPS: { label: string; category: string }[] = [
+  { label: 'Expected Standard', category: 'expected' },
+  { label: 'Higher Standard', category: 'higher' },
+  { label: 'Progress Scores', category: 'progress' },
+  { label: 'Average Scores', category: 'average' },
+  { label: 'Gender Performance', category: 'gender' },
+  { label: 'Equity (Disadvantaged)', category: 'equity' },
+  { label: 'School Context', category: 'context' },
+  { label: 'Absence', category: 'absence' },
+  { label: '3-Year Trends', category: 'trends' },
+];
+
+const SECONDARY_OPTGROUPS: { label: string; category: string }[] = [
+  { label: 'GCSE Performance', category: 'gcse' },
+];
+
+export const PRIMARY_CATEGORIES = PRIMARY_OPTGROUPS.map((g) => g.category);
+export const SECONDARY_CATEGORIES = SECONDARY_OPTGROUPS.map((g) => g.category);
+
+const PROGRESS_CI: Record<string, [string, string]> = {
+  reading_progress: ['reading_progress_lower_ci', 'reading_progress_upper_ci'],
+  writing_progress: ['writing_progress_lower_ci', 'writing_progress_upper_ci'],
+  maths_progress: ['maths_progress_lower_ci', 'maths_progress_upper_ci'],
+};
+
+const BAND_LABEL = { above: 'Above average', average: 'Average', below: 'Below average' } as const;
+
+export function TrendsExplorer({
+  schools,
+  data,
+  metrics,
+  initialMetric,
+  isPrimaryPhase,
+  nationalAverages,
+}: {
+  schools: School[];
+  data: Record<string, ComparisonData>;
+  metrics: MetricDefinition[];
+  initialMetric: string;
+  isPrimaryPhase: boolean;
+  nationalAverages?: NationalAverages;
+}) {
+  const [metric, setMetric] = useState(initialMetric);
+
+  const allowedCategories = isPrimaryPhase ? PRIMARY_CATEGORIES : SECONDARY_CATEGORIES;
+  const optgroups = isPrimaryPhase ? PRIMARY_OPTGROUPS : SECONDARY_OPTGROUPS;
+  const filteredMetrics = metrics.filter((m) => allowedCategories.includes(m.category));
+  const metricDef = metrics.find((m) => m.key === metric);
+  const metricLabel = metricDef?.label || metric;
+
+  const nationalByYear: Record<number, number | null> = {};
+  for (const entry of nationalAverages?.by_year ?? []) {
+    const block = isPrimaryPhase ? entry.primary : entry.secondary;
+    nationalByYear[entry.year] = block?.[metric] ?? null;
+  }
+
+  const years = [
+    ...new Set(
+      schools.flatMap(
+        (school) => data[String(school.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [],
+      ),
+    ),
+  ].sort((a, b) => a - b);
+
+  const handleMetricChange = (next: string) => {
+    track('compare_metric_changed', { metric: next, phase: isPrimaryPhase ? 'primary' : 'secondary' });
+    setMetric(next);
+  };
+
+  const ciKeys = PROGRESS_CI[metric];
+
+  return (
+    <Section
+      title="Explore trends"
+      how="The full year-by-year explorer for any measure, with the England average as a dashed reference line where official figures exist."
+    >
+      <details className={styles.explore} open>
+        <summary>Year-by-year trends</summary>
+        <div className={styles.inner}>
+          <div className={styles.picker}>
+            <label htmlFor="trends-metric-select">Measure:</label>
+            <select
+              id="trends-metric-select"
+              value={metric}
+              onChange={(e) => handleMetricChange(e.target.value)}
+            >
+              {optgroups.map(({ label, category }) => {
+                const group = filteredMetrics.filter((m) => m.category === category);
+                if (group.length === 0) return null;
+                return (
+                  <optgroup key={category} label={label}>
+                    {group.map((m) => (
+                      <option key={m.key} value={m.key}>
+                        {m.label}
+                      </option>
+                    ))}
+                  </optgroup>
+                );
+              })}
+            </select>
+            {metricDef?.description && <span className={styles.desc}>{metricDef.description}</span>}
+          </div>
+
+          {metric.includes('progress') && (
+            <p className={styles.progressNote}>
+              Progress scores measure pupils&apos; progress from KS1 to KS2. A score of 0 equals the
+              national average. DfE stopped publishing KS2 progress after 2022/23 (no KS1 baseline);
+              bands use DfE&apos;s confidence intervals, not the raw score alone.
+            </p>
+          )}
+
+          <div className={styles.chartBox}>
+            <ComparisonChart
+              comparisonData={data}
+              schools={schools}
+              metric={metric}
+              metricLabel={metricLabel}
+              nationalByYear={nationalByYear}
+            />
+          </div>
+
+          {years.length > 0 && (
+            <div className={styles.tableWrapper}>
+              <table className={styles.table}>
+                <thead>
+                  <tr>
+                    <th>Year</th>
+                    {schools.map((school) => (
+                      <th key={school.urn}>{school.school_name}</th>
+                    ))}
+                  </tr>
+                </thead>
+                <tbody>
+                  {years.map((year) => (
+                    <tr key={year}>
+                      <td className={styles.yearCell}>{formatAcademicYear(year)}</td>
+                      {schools.map((school) => {
+                        const row = data[String(school.urn)]?.yearly_data.find(
+                          (d) => Math.trunc(d.year) === year,
+                        ) as (Record<string, unknown> & { year: number }) | undefined;
+                        const value = row?.[metric];
+                        if (typeof value !== 'number') return <td key={school.urn}>–</td>;
+                        const band = ciKeys
+                          ? progressBand(
+                              value,
+                              (row?.[ciKeys[0]] as number | null) ?? null,
+                              (row?.[ciKeys[1]] as number | null) ?? null,
+                            )
+                          : null;
+                        return (
+                          <td key={school.urn}>
+                            {formatMetricValue(value, metricKind(metric))}{' '}
+                            {band && (
+                              <Chip tone={band === 'above' ? 'good' : band === 'below' ? 'warn' : 'neutral'}>
+                                {BAND_LABEL[band]}
+                              </Chip>
+                            )}
+                          </td>
+                        );
+                      })}
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+          )}
+        </div>
+      </details>
+    </Section>
+  );
+}

nextjs-app/lib/compareChartData.ts

@@ -0,0 +1,102 @@
+/**
+ * Pure series-building for the comparison trend chart, extracted from
+ * ComparisonChart so it is unit-testable without a canvas.
+ *
+ * Chart truthfulness rules (spec §8.1): every academic year between the
+ * first and last data point appears on the axis — cancelled test years
+ * (2019/20, 2020/21) and the unpublished 2021/22 school-level year render
+ * as real gaps, never as compressed time; school lines never bridge gaps.
+ */
+
+import type { ComparisonData } from './types';
+
+/** 201819 → 201920 (academic-year arithmetic on YYYYYY codes). */
+function nextAcademicYear(year: number): number {
+  const start = Math.floor(year / 100);
+  const end = year % 100;
+  return (start + 1) * 100 + (end + 1);
+}
+
+/** Every academic year from min(years) to max(years), inclusive. */
+export function fillAcademicYears(years: number[]): number[] {
+  if (years.length === 0) return [];
+  const ints = [...new Set(years.map((y) => Math.trunc(y)))].sort((a, b) => a - b);
+  const out: number[] = [];
+  let y = ints[0];
+  const last = ints[ints.length - 1];
+  while (y <= last && out.length < 50) {
+    out.push(y);
+    y = nextAcademicYear(y);
+  }
+  return out;
+}
+
+export interface CompareChartSeries {
+  label: string;
+  data: Array<number | null>;
+  /** Index into CHART_COLORS / point styles. */
+  schoolIndex: number;
+  spanGaps: false;
+}
+
+export interface EnglandSeries {
+  label: 'England average';
+  data: Array<number | null>;
+  borderDash: [number, number];
+  spanGaps: false;
+}
+
+export interface CompareChart {
+  years: number[];
+  schoolDatasets: CompareChartSeries[];
+  englandDataset: EnglandSeries | null;
+  /** True when England published a 2021/22 figure but no school has one —
+   * the UI shows: "DfE didn't publish school-level figures for 2021/22". */
+  showUnpublished202122Note: boolean;
+}
+
+export function buildCompareChart(
+  comparisonData: Record<string, ComparisonData>,
+  schools: Array<{ urn: number; school_name: string }>,
+  metric: string,
+  nationalByYear?: Record<number, number | null | undefined>,
+): CompareChart {
+  const rawYears = schools.flatMap(
+    (s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => Math.trunc(d.year)) ?? [],
+  );
+  const years = fillAcademicYears(rawYears);
+
+  const schoolDatasets: CompareChartSeries[] = schools.map((school, schoolIndex) => {
+    const rows = comparisonData[String(school.urn)]?.yearly_data ?? [];
+    const byYear = new Map<number, Record<string, unknown>>();
+    for (const row of rows) byYear.set(Math.trunc(row.year), row as unknown as Record<string, unknown>);
+    return {
+      label: school.school_name,
+      data: years.map((year) => {
+        const v = byYear.get(year)?.[metric];
+        return typeof v === 'number' && !Number.isNaN(v) ? v : null;
+      }),
+      schoolIndex,
+      spanGaps: false,
+    };
+  });
+
+  let englandDataset: EnglandSeries | null = null;
+  if (nationalByYear) {
+    const data = years.map((year) => {
+      const v = nationalByYear[year];
+      return typeof v === 'number' && !Number.isNaN(v) ? v : null;
+    });
+    if (data.some((v) => v != null)) {
+      englandDataset = { label: 'England average', data, borderDash: [5, 4], spanGaps: false };
+    }
+  }
+
+  const idx202122 = years.indexOf(202122);
+  const showUnpublished202122Note =
+    idx202122 >= 0 &&
+    englandDataset?.data[idx202122] != null &&
+    schoolDatasets.every((ds) => ds.data[idx202122] == null);
+
+  return { years, schoolDatasets, englandDataset, showUnpublished202122Note };
+}