diff --git a/backend/app.py b/backend/app.py index e01c4c5..b935af8 100644 --- a/backend/app.py +++ b/backend/app.py @@ -772,93 +772,101 @@ async def get_la_averages(request: Request): return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}} +_KS2_NATIONAL_METRICS = [ + "rwm_expected_pct", "rwm_high_pct", + "reading_expected_pct", "writing_expected_pct", "maths_expected_pct", + "gps_expected_pct", "gps_high_pct", "science_expected_pct", + "reading_avg_score", "maths_avg_score", "gps_avg_score", + "reading_progress", "writing_progress", "maths_progress", + "overall_absence_pct", "persistent_absence_pct", + "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct", +] +_KS4_NATIONAL_METRICS = [ + "attainment_8_score", "progress_8_score", + "english_maths_standard_pass_pct", "english_maths_strong_pass_pct", + "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct", + "ebacc_avg_score", "gcse_grade_91_pct", +] + + def _national_averages_payload(df: pd.DataFrame) -> dict: """National-averages payload shared by /api/national-averages and - /api/compare. Official DfE KS2 figures come from the mart table; - KS4 figures are computed from our dataset (no DfE dataset yet).""" + /api/compare. + + Both series are persisted marts computed at import time: official DfE + KS2 figures (fact_ks2_national_averages) and dataset-computed KS4 + averages (fact_ks4_national_averages) — the API never aggregates the + performance dataframe per request. If the KS4 mart hasn't been built + yet (deploy lands before the next DAG run), fall back to computing the + latest year only — a single-year scan, never the historical loop. + """ if df.empty: return {"primary": {}, "secondary": {}} - ks2_metrics = [ - "rwm_expected_pct", "rwm_high_pct", - "reading_expected_pct", "writing_expected_pct", "maths_expected_pct", - "gps_expected_pct", "gps_high_pct", "science_expected_pct", - "reading_avg_score", "maths_avg_score", "gps_avg_score", - "reading_progress", "writing_progress", "maths_progress", - "overall_absence_pct", "persistent_absence_pct", - "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct", - ] - ks4_metrics = [ - "attainment_8_score", "progress_8_score", - "english_maths_standard_pass_pct", "english_maths_strong_pass_pct", - "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct", - "ebacc_avg_score", "gcse_grade_91_pct", - ] + latest_year = int(df["year"].max()) - def _means(sub_df, metric_list): + from . import database + from .models import Ks2NationalAverage, Ks4NationalAverage + + def _row_metrics(row, metric_list): out = {} for col in metric_list: - if col in sub_df.columns: - val = sub_df[col].dropna() - if len(val) > 0: - out[col] = round(float(val.mean()), 2) + val = getattr(row, col, None) + if val is not None: + out[col] = val return out - latest_year = int(df["year"].max()) - df_latest = df[df["year"] == latest_year] - - # Primary: schools where KS2 data is non-null - primary_df = df_latest[df_latest["rwm_expected_pct"].notna()] - # Secondary: schools where KS4 data is non-null - secondary_df = df_latest[df_latest["attainment_8_score"].notna()] - - latest_primary = _means(primary_df, ks2_metrics) - latest_secondary = _means(secondary_df, ks4_metrics) - - # Per-year KS2 primary averages: use official DfE figures from the mart table. - # Per-year KS4 secondary averages: computed from our dataset (no DfE dataset yet). - from . import database - from .models import Ks2NationalAverage - - by_year = [] + ks2_rows: list = [] + ks4_rows: list = [] db = None try: db = database.SessionLocal() - nat_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all() - # Build a lookup of computed secondary averages per year as fallback - secondary_by_year = {} - for yr in sorted(df["year"].dropna().unique()): - yr = int(yr) - df_yr = df[df["year"] == yr] - secondary_by_year[yr] = _means( - df_yr[df_yr["attainment_8_score"].notna()], ks4_metrics - ) - # Merge: official KS2 figures + computed KS4 figures per year - ks2_years = {r.year for r in nat_rows} - all_years = sorted(ks2_years | set(secondary_by_year.keys())) - nat_lookup = {r.year: r for r in nat_rows} - for yr in all_years: - primary_yr: dict = {} - if yr in nat_lookup: - r = nat_lookup[yr] - for col in ks2_metrics: - val = getattr(r, col, None) - if val is not None: - primary_yr[col] = val - by_year.append({ - "year": yr, - "primary": primary_yr, - "secondary": secondary_by_year.get(yr, {}), - }) + try: + ks2_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all() + except Exception: + db.rollback() + try: + ks4_rows = db.query(Ks4NationalAverage).order_by(Ks4NationalAverage.year).all() + except Exception: + db.rollback() + except Exception: + pass finally: if db is not None: db.close() - # Update latest_primary with official DfE figure for the latest year if available - if by_year: - latest_official = next((e["primary"] for e in reversed(by_year) if e["primary"]), None) - if latest_official: - latest_primary = latest_official + primary_by_year = {r.year: _row_metrics(r, _KS2_NATIONAL_METRICS) for r in ks2_rows} + secondary_by_year = {r.year: _row_metrics(r, _KS4_NATIONAL_METRICS) for r in ks4_rows} + + if not any(secondary_by_year.values()): + # KS4 mart missing/empty: compute the latest year only. + df_latest = df[df["year"] == latest_year] + sec = ( + df_latest[df_latest["attainment_8_score"].notna()] + if "attainment_8_score" in df_latest.columns + else df_latest.iloc[0:0] + ) + vals = {} + for col in _KS4_NATIONAL_METRICS: + if col in sec.columns: + v = sec[col].dropna() + if len(v) > 0: + vals[col] = round(float(v.mean()), 2) + if vals: + secondary_by_year[latest_year] = vals + + all_years = sorted(set(primary_by_year) | set(secondary_by_year)) + by_year = [ + { + "year": yr, + "primary": primary_by_year.get(yr, {}), + "secondary": secondary_by_year.get(yr, {}), + } + for yr in all_years + ] + + latest_primary = next((e["primary"] for e in reversed(by_year) if e["primary"]), {}) + latest_secondary = next((e["secondary"] for e in reversed(by_year) if e["secondary"]), {}) return { "year": latest_year, diff --git a/backend/models.py b/backend/models.py index 8f97a16..b2325d9 100644 --- a/backend/models.py +++ b/backend/models.py @@ -231,6 +231,23 @@ class FactFinance(Base): premises_cost_pct = Column(Float) +class Ks4NationalAverage(Base): + """Computed national KS4 averages (from our dataset) — one row per year.""" + __tablename__ = "fact_ks4_national_averages" + __table_args__ = MARTS + + year = Column(Integer, primary_key=True) + attainment_8_score = Column(Float) + progress_8_score = Column(Float) + english_maths_standard_pass_pct = Column(Float) + english_maths_strong_pass_pct = Column(Float) + ebacc_entry_pct = Column(Float) + ebacc_standard_pass_pct = Column(Float) + ebacc_strong_pass_pct = Column(Float) + ebacc_avg_score = Column(Float) + gcse_grade_91_pct = Column(Float) + + class Ks2NationalAverage(Base): """Official DfE KS2 national headline averages — one row per academic year.""" __tablename__ = "fact_ks2_national_averages" diff --git a/backend/tests/test_national_averages_marts.py b/backend/tests/test_national_averages_marts.py new file mode 100644 index 0000000..b08c6df --- /dev/null +++ b/backend/tests/test_national_averages_marts.py @@ -0,0 +1,93 @@ +"""_national_averages_payload reads persisted marts (computed at import +time) — it must never loop the dataframe per year. The only dataframe work +allowed is the single-latest-year KS4 fallback for the window between a +deploy and the next DAG run.""" + +import numpy as np +import pandas as pd +import pytest + +LATEST = 202425 + + +def _df(): + return pd.DataFrame( + [ + dict(year=202324, attainment_8_score=40.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=50.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=30.0, rwm_expected_pct=np.nan), + dict(year=LATEST, attainment_8_score=np.nan, rwm_expected_pct=80.0), + ] + ) + + +class _Ks2Row: + year = LATEST + rwm_expected_pct = 62.1 + gps_expected_pct = 72.0 + + +class _Ks4Row: + year = LATEST + attainment_8_score = 46.5 + progress_8_score = -0.02 + + +class _StubSession: + """Returns KS2 rows for the first query and KS4 rows for the second — + mirroring the payload's query order.""" + + def __init__(self): + self.calls = 0 + + def query(self, model): + self._model = model.__name__ + return self + + def order_by(self, *a): + return self + + def all(self): + return [_Ks2Row()] if self._model == "Ks2NationalAverage" else [_Ks4Row()] + + def close(self): + pass + + +class _Ks4MissingSession(_StubSession): + def all(self): + if self._model == "Ks4NationalAverage": + raise RuntimeError("relation does not exist") + return [_Ks2Row()] + + def rollback(self): + pass + + +@pytest.fixture() +def payload(monkeypatch): + from backend import app as app_module + from backend import database as database_module + + def _run(session_cls): + monkeypatch.setattr(database_module, "SessionLocal", session_cls) + return app_module._national_averages_payload(_df()) + + return _run + + +def test_ks4_averages_come_from_the_mart_not_the_dataframe(payload): + body = payload(_StubSession) + # Mart value (46.5), NOT the dataframe mean of (50+30)/2 = 40.0 + assert body["secondary"]["attainment_8_score"] == 46.5 + assert body["primary"]["rwm_expected_pct"] == 62.1 + assert body["by_year"][-1]["secondary"]["progress_8_score"] == -0.02 + + +def test_missing_ks4_mart_falls_back_to_latest_year_only(payload): + body = payload(_Ks4MissingSession) + # Fallback computes the latest year from the df: mean(50, 30) = 40.0 + assert body["secondary"]["attainment_8_score"] == 40.0 + # ...and only the latest year — no historical KS4 loop + ks4_years = [e["year"] for e in body["by_year"] if e["secondary"]] + assert ks4_years == [LATEST] diff --git a/nextjs-app/app/compare/page.tsx b/nextjs-app/app/compare/page.tsx index 31689a5..0b2b4e4 100644 --- a/nextjs-app/app/compare/page.tsx +++ b/nextjs-app/app/compare/page.tsx @@ -32,26 +32,24 @@ export default async function ComparePage({ searchParams }: ComparePageProps) { const selectedMetric = metricParam || 'rwm_expected_pct'; try { - // Fetch comparison data if URNs provided - let comparisonData = null; - if (urns.length > 0) { - try { - const response = await fetchComparison(urnsParam!); - comparisonData = response.comparison; - } catch (error) { - console.error('Failed to fetch comparison:', error); - } - } + // Fetch comparison + metrics in parallel — they are independent. + const [comparisonResponse, metricsResponse] = await Promise.all([ + urns.length > 0 + ? fetchComparison(urnsParam!).catch((error) => { + console.error('Failed to fetch comparison:', error); + return null; + }) + : Promise.resolve(null), + fetchMetrics(), + ]); - // Fetch available metrics - const metricsResponse = await fetchMetrics(); - - // Metrics is already an array const metricsArray = metricsResponse?.metrics || []; return ( | null; + initialNationalAverages?: NationalAverages; + initialBenchmarks?: Benchmarks; initialUrns: number[]; metrics: MetricDefinition[]; selectedMetric: string; @@ -42,6 +44,8 @@ interface ComparisonViewProps { export function ComparisonView({ initialData, + initialNationalAverages, + initialBenchmarks, initialUrns, metrics, selectedMetric: initialMetric, @@ -54,8 +58,10 @@ export function ComparisonView({ const [selectedMetric, setSelectedMetric] = useState(initialMetric); const [isModalOpen, setIsModalOpen] = useState(false); const [comparisonData, setComparisonData] = useState(initialData); - const [nationalAverages, setNationalAverages] = useState(); - const [benchmarks, setBenchmarks] = useState(); + const [nationalAverages, setNationalAverages] = useState( + initialNationalAverages, + ); + const [benchmarks, setBenchmarks] = useState(initialBenchmarks); const [shareConfirm, setShareConfirm] = useState(false); const [comparePhase, setComparePhase] = useState<'primary' | 'secondary'>('primary'); // Tracks whether the user has explicitly clicked a phase tab. @@ -81,13 +87,16 @@ export function ComparisonView({ } }, [isInitialized]); // eslint-disable-line react-hooks/exhaustive-deps - // Sync URL with selected schools + metric, and (re)fetch the comparison. + const urnKey = selectedSchools.map((s) => s.urn).join(','); + + // Sync the URL with the selection + metric. Pure navigation state — no + // fetching here: metric changes are presentational (the data is already + // client-side) and must not refire the comparison request. useEffect(() => { - const urns = selectedSchools.map((s) => s.urn).join(','); const params = new URLSearchParams(searchParams); - if (urns) { - params.set('urns', urns); + if (urnKey) { + params.set('urns', urnKey); } else { params.delete('urns'); } @@ -96,26 +105,41 @@ export function ComparisonView({ const newUrl = `${pathname}?${params.toString()}`; router.replace(newUrl, { scroll: false }); + }, [urnKey, selectedMetric, pathname, searchParams, router]); - if (selectedSchools.length > 0) { - fetchComparison(urns, { cache: 'no-store' }) - .then((data) => { - setComparisonData(data.comparison); - setNationalAverages(data.national_averages); - setBenchmarks(data.benchmarks); - }) - .catch((err) => { - // Keep whatever we already have (SSR data or a previous fetch) rather - // than blanking the page — a transient refetch failure shouldn't - // destroy a working comparison the user is looking at. - console.error('Failed to fetch comparison:', err); - }); - } else { + // Fetch only when the school set changes. The very first run is skipped + // when the SSR payload already covers the current set — no double-fetch + // of data the server just rendered. + const firstFetchRef = useRef(true); + useEffect(() => { + if (!urnKey) { setComparisonData(null); setNationalAverages(undefined); setBenchmarks(undefined); + return; } - }, [selectedSchools, selectedMetric, pathname, searchParams, router]); + + if (firstFetchRef.current) { + firstFetchRef.current = false; + const ssrUrns = new Set(Object.keys(initialData ?? {})); + const covered = urnKey.split(',').every((urn) => ssrUrns.has(urn)); + if (covered && ssrUrns.size > 0) return; + } + + fetchComparison(urnKey, { cache: 'no-store' }) + .then((data) => { + setComparisonData(data.comparison); + setNationalAverages(data.national_averages); + setBenchmarks(data.benchmarks); + }) + .catch((err) => { + // Keep whatever we already have (SSR data or a previous fetch) rather + // than blanking the page — a transient refetch failure shouldn't + // destroy a working comparison the user is looking at. + console.error('Failed to fetch comparison:', err); + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [urnKey]); // Classify schools by phase using comparison data const classifySchool = (school: School): 'primary' | 'secondary' => { diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index bd4072e..9fc7860 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -160,6 +160,12 @@ models: - name: year tests: [not_null, unique] + - name: fact_ks4_national_averages + description: Computed national KS4 averages (means across state schools in our dataset — not official DfE figures) — one row per academic year + columns: + - name: year + tests: [not_null, unique] + - name: fact_deprivation description: IDACI deprivation index — one row per URN columns: diff --git a/pipeline/transform/models/marts/fact_ks4_national_averages.sql b/pipeline/transform/models/marts/fact_ks4_national_averages.sql new file mode 100644 index 0000000..5022332 --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks4_national_averages.sql @@ -0,0 +1,25 @@ +{{ config(materialized='table') }} + +-- Mart: Computed national KS4 averages — one row per academic year. +-- Unlike fact_ks2_national_averages (official DfE figures), DfE publishes no +-- KS4 national-headline dataset we ingest yet, so these are means computed +-- across the state schools in our dataset. Computed once at build time so the +-- API never has to aggregate the full performance table per request. +-- Semantics match the API's previous per-request computation: rows where +-- attainment_8_score is non-null; per-column means ignore NULLs. + +select + year, + round(avg(attainment_8_score)::numeric, 2) as attainment_8_score, + round(avg(progress_8_score)::numeric, 2) as progress_8_score, + round(avg(english_maths_standard_pass_pct)::numeric, 2) as english_maths_standard_pass_pct, + round(avg(english_maths_strong_pass_pct)::numeric, 2) as english_maths_strong_pass_pct, + round(avg(ebacc_entry_pct)::numeric, 2) as ebacc_entry_pct, + round(avg(ebacc_standard_pass_pct)::numeric, 2) as ebacc_standard_pass_pct, + round(avg(ebacc_strong_pass_pct)::numeric, 2) as ebacc_strong_pass_pct, + round(avg(ebacc_avg_score)::numeric, 2) as ebacc_avg_score, + round(avg(gcse_grade_91_pct)::numeric, 2) as gcse_grade_91_pct +from {{ ref('fact_ks4_performance') }} +where attainment_8_score is not null +group by year +order by year