perf(api): persist KS4 national averages as a mart; stop per-request aggregation

fact_ks4_national_averages is computed once at dbt build time (covered by the EES DAG's stg_ees_ks4+ selector). _national_averages_payload now reads both national-averages marts instead of scanning the performance dataframe per year on every /api/compare request (~250ms saved per call). Fallback for the deploy-before-DAG window computes the latest year only. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB
2026-07-14 13:05:03 +01:00
parent 9990f540f7
commit 52f8994401
5 changed files with 219 additions and 70 deletions
@@ -772,93 +772,101 @@ async def get_la_averages(request: Request):
    return {"year": latest_year, "secondary": {"attainment_8_by_la": la_avg}}


+_KS2_NATIONAL_METRICS = [
+    "rwm_expected_pct", "rwm_high_pct",
+    "reading_expected_pct", "writing_expected_pct", "maths_expected_pct",
+    "gps_expected_pct", "gps_high_pct", "science_expected_pct",
+    "reading_avg_score", "maths_avg_score", "gps_avg_score",
+    "reading_progress", "writing_progress", "maths_progress",
+    "overall_absence_pct", "persistent_absence_pct",
+    "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct",
+]
+_KS4_NATIONAL_METRICS = [
+    "attainment_8_score", "progress_8_score",
+    "english_maths_standard_pass_pct", "english_maths_strong_pass_pct",
+    "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct",
+    "ebacc_avg_score", "gcse_grade_91_pct",
+]
+
+
 def _national_averages_payload(df: pd.DataFrame) -> dict:
    """National-averages payload shared by /api/national-averages and
-    /api/compare. Official DfE KS2 figures come from the mart table;
-    KS4 figures are computed from our dataset (no DfE dataset yet)."""
+    /api/compare.
+
+    Both series are persisted marts computed at import time: official DfE
+    KS2 figures (fact_ks2_national_averages) and dataset-computed KS4
+    averages (fact_ks4_national_averages) — the API never aggregates the
+    performance dataframe per request. If the KS4 mart hasn't been built
+    yet (deploy lands before the next DAG run), fall back to computing the
+    latest year only — a single-year scan, never the historical loop.
+    """
    if df.empty:
        return {"primary": {}, "secondary": {}}

-    ks2_metrics = [
-        "rwm_expected_pct", "rwm_high_pct",
-        "reading_expected_pct", "writing_expected_pct", "maths_expected_pct",
-        "gps_expected_pct", "gps_high_pct", "science_expected_pct",
-        "reading_avg_score", "maths_avg_score", "gps_avg_score",
-        "reading_progress", "writing_progress", "maths_progress",
-        "overall_absence_pct", "persistent_absence_pct",
-        "disadvantaged_gap", "disadvantaged_pct", "sen_support_pct", "eal_pct",
-    ]
-    ks4_metrics = [
-        "attainment_8_score", "progress_8_score",
-        "english_maths_standard_pass_pct", "english_maths_strong_pass_pct",
-        "ebacc_entry_pct", "ebacc_standard_pass_pct", "ebacc_strong_pass_pct",
-        "ebacc_avg_score", "gcse_grade_91_pct",
-    ]
+    latest_year = int(df["year"].max())

-    def _means(sub_df, metric_list):
+    from . import database
+    from .models import Ks2NationalAverage, Ks4NationalAverage
+
+    def _row_metrics(row, metric_list):
        out = {}
        for col in metric_list:
-            if col in sub_df.columns:
-                val = sub_df[col].dropna()
-                if len(val) > 0:
-                    out[col] = round(float(val.mean()), 2)
+            val = getattr(row, col, None)
+            if val is not None:
+                out[col] = val
        return out

-    latest_year = int(df["year"].max())
-    df_latest = df[df["year"] == latest_year]
-
-    # Primary: schools where KS2 data is non-null
-    primary_df = df_latest[df_latest["rwm_expected_pct"].notna()]
-    # Secondary: schools where KS4 data is non-null
-    secondary_df = df_latest[df_latest["attainment_8_score"].notna()]
-
-    latest_primary = _means(primary_df, ks2_metrics)
-    latest_secondary = _means(secondary_df, ks4_metrics)
-
-    # Per-year KS2 primary averages: use official DfE figures from the mart table.
-    # Per-year KS4 secondary averages: computed from our dataset (no DfE dataset yet).
-    from . import database
-    from .models import Ks2NationalAverage
-
-    by_year = []
+    ks2_rows: list = []
+    ks4_rows: list = []
    db = None
    try:
        db = database.SessionLocal()
-        nat_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all()
-        # Build a lookup of computed secondary averages per year as fallback
-        secondary_by_year = {}
-        for yr in sorted(df["year"].dropna().unique()):
-            yr = int(yr)
-            df_yr = df[df["year"] == yr]
-            secondary_by_year[yr] = _means(
-                df_yr[df_yr["attainment_8_score"].notna()], ks4_metrics
-            )
-        # Merge: official KS2 figures + computed KS4 figures per year
-        ks2_years = {r.year for r in nat_rows}
-        all_years = sorted(ks2_years | set(secondary_by_year.keys()))
-        nat_lookup = {r.year: r for r in nat_rows}
-        for yr in all_years:
-            primary_yr: dict = {}
-            if yr in nat_lookup:
-                r = nat_lookup[yr]
-                for col in ks2_metrics:
-                    val = getattr(r, col, None)
-                    if val is not None:
-                        primary_yr[col] = val
-            by_year.append({
-                "year": yr,
-                "primary": primary_yr,
-                "secondary": secondary_by_year.get(yr, {}),
-            })
+        try:
+            ks2_rows = db.query(Ks2NationalAverage).order_by(Ks2NationalAverage.year).all()
+        except Exception:
+            db.rollback()
+        try:
+            ks4_rows = db.query(Ks4NationalAverage).order_by(Ks4NationalAverage.year).all()
+        except Exception:
+            db.rollback()
+    except Exception:
+        pass
    finally:
        if db is not None:
            db.close()

-    # Update latest_primary with official DfE figure for the latest year if available
-    if by_year:
-        latest_official = next((e["primary"] for e in reversed(by_year) if e["primary"]), None)
-        if latest_official:
-            latest_primary = latest_official
+    primary_by_year = {r.year: _row_metrics(r, _KS2_NATIONAL_METRICS) for r in ks2_rows}
+    secondary_by_year = {r.year: _row_metrics(r, _KS4_NATIONAL_METRICS) for r in ks4_rows}
+
+    if not any(secondary_by_year.values()):
+        # KS4 mart missing/empty: compute the latest year only.
+        df_latest = df[df["year"] == latest_year]
+        sec = (
+            df_latest[df_latest["attainment_8_score"].notna()]
+            if "attainment_8_score" in df_latest.columns
+            else df_latest.iloc[0:0]
+        )
+        vals = {}
+        for col in _KS4_NATIONAL_METRICS:
+            if col in sec.columns:
+                v = sec[col].dropna()
+                if len(v) > 0:
+                    vals[col] = round(float(v.mean()), 2)
+        if vals:
+            secondary_by_year[latest_year] = vals
+
+    all_years = sorted(set(primary_by_year) | set(secondary_by_year))
+    by_year = [
+        {
+            "year": yr,
+            "primary": primary_by_year.get(yr, {}),
+            "secondary": secondary_by_year.get(yr, {}),
+        }
+        for yr in all_years
+    ]
+
+    latest_primary = next((e["primary"] for e in reversed(by_year) if e["primary"]), {})
+    latest_secondary = next((e["secondary"] for e in reversed(by_year) if e["secondary"]), {})

    return {
        "year": latest_year,