Refactoring and bug fixes

Dockerfile

@@ -26,6 +26,6 @@ COPY data/ ./data/
 # Expose the application port
 EXPOSE 80
 
-# Run the application
+# Run the application (using module import)
-CMD ["uvicorn", "backend.app:app", "--host", "0.0.0.0", "--port", "80"]
+CMD ["python", "-m", "uvicorn", "backend.app:app", "--host", "0.0.0.0", "--port", "80"]
 

backend/__init__.py

@@ -0,0 +1,2 @@
+# Backend package
+

backend/app.py

@@ -4,284 +4,83 @@ Serves primary school (KS2) performance data for comparing schools.
 Uses real data from UK Government Compare School Performance downloads.
 """
 
+from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException, Query
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import FileResponse
 from fastapi.middleware.cors import CORSMiddleware
-import pandas as pd
-import numpy as np
-from pathlib import Path
 from typing import Optional
-import os
-import re
 
-# No longer filtering by specific LA codes - load all available schools
+from .config import settings
+from .schemas import METRIC_DEFINITIONS, RANKING_COLUMNS, SCHOOL_COLUMNS
+from .data_loader import load_school_data, clear_cache
+from .utils import clean_for_json
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan - startup and shutdown events."""
+    # Startup: pre-load data
+    print("Starting up: Loading school data...")
+    load_school_data()
+    print("Data loaded successfully.")
+    
+    yield  # Application runs here
+    
+    # Shutdown: cleanup if needed
+    print("Shutting down...")
+
 
 app = FastAPI(
     title="SchoolCompare API",
     description="API for comparing primary school (KS2) performance data - schoolcompare.co.uk",
-    version="1.0.0"
+    version="2.0.0",
+    lifespan=lifespan,
 )
 
-# CORS middleware for development
+# CORS middleware with configurable origins
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=settings.allowed_origins,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
-# Data directory
-DATA_DIR = Path(__file__).parent.parent / "data"
-FRONTEND_DIR = Path(__file__).parent.parent / "frontend"
-
-# Cache for loaded data - cleared on reload (updated for 2016-2017 data)
-_data_cache: Optional[pd.DataFrame] = None
-
-
-def convert_to_native(value):
-    """Convert numpy types to native Python types for JSON serialization."""
-    if pd.isna(value):
-        return None
-    if isinstance(value, (np.integer,)):
-        return int(value)
-    if isinstance(value, (np.floating,)):
-        if np.isnan(value) or np.isinf(value):
-            return None
-        return float(value)
-    if isinstance(value, np.ndarray):
-        return value.tolist()
-    if value == "SUPP" or value == "NE" or value == "NA" or value == "NP":
-        return None
-    return value
-
-
-def clean_for_json(df: pd.DataFrame) -> list:
-    """Convert DataFrame to list of dicts, replacing NaN/inf with None for JSON serialization."""
-    records = df.to_dict(orient="records")
-    cleaned = []
-    for record in records:
-        clean_record = {}
-        for key, value in record.items():
-            clean_record[key] = convert_to_native(value)
-        cleaned.append(clean_record)
-    return cleaned
-
-
-def parse_numeric(value):
-    """Parse a value to numeric, handling SUPP, NE, NA, %, etc."""
-    if pd.isna(value):
-        return None
-    if isinstance(value, (int, float)):
-        if np.isnan(value) or np.isinf(value):
-            return None
-        return value
-    if isinstance(value, str):
-        value = value.strip()
-        if value in ["SUPP", "NE", "NA", "NP", "NEW", "LOW", ""]:
-            return None
-        # Remove % sign if present
-        if value.endswith('%'):
-            value = value[:-1]
-        try:
-            return float(value)
-        except ValueError:
-            return None
-    return None
-
-
-def extract_year_from_folder(folder_name: str) -> Optional[int]:
-    """Extract the end year from folder name like '2023-2024' -> 2024."""
-    match = re.search(r'(\d{4})-(\d{4})', folder_name)
-    if match:
-        return int(match.group(2))
-    return None
-
-
-def load_school_data() -> pd.DataFrame:
-    """Load and combine all school data from CSV files in year folders."""
-    global _data_cache
-    
-    if _data_cache is not None:
-        return _data_cache
-    
-    all_data = []
-    
-    # Look for year folders in data directory
-    if DATA_DIR.exists():
-        for year_folder in DATA_DIR.iterdir():
-            if year_folder.is_dir() and re.match(r'\d{4}-\d{4}', year_folder.name):
-                year = extract_year_from_folder(year_folder.name)
-                if year is None:
-                    continue
-                
-                # Look for KS2 data file
-                ks2_file = year_folder / "england_ks2final.csv"
-                if ks2_file.exists():
-                    try:
-                        print(f"Loading data from {ks2_file}")
-                        df = pd.read_csv(ks2_file, low_memory=False)
-                        
-                        # Handle both string and integer columns
-                        if 'LEA' in df.columns and df['LEA'].dtype == 'object':
-                            df['LEA'] = pd.to_numeric(df['LEA'], errors='coerce')
-                        if 'URN' in df.columns and df['URN'].dtype == 'object':
-                            df['URN'] = pd.to_numeric(df['URN'], errors='coerce')
-                        
-                        # Filter to schools only (RECTYPE == 1 means school level data)
-                        if 'RECTYPE' in df.columns:
-                            df = df[df['RECTYPE'] == 1]
-                        
-                        # Add year and local authority name from LANAME column
-                        df['year'] = year
-                        if 'LANAME' in df.columns:
-                            df['local_authority'] = df['LANAME']
-                        elif 'LEA' in df.columns:
-                            df['local_authority'] = df['LEA'].astype(str)
-                        
-                        # Standardize column names for our API
-                        df = df.rename(columns={
-                            'URN': 'urn',
-                            'SCHNAME': 'school_name',
-                            'ADDRESS1': 'address1',
-                            'ADDRESS2': 'address2',
-                            'TOWN': 'town',
-                            'PCODE': 'postcode',
-                            'NFTYPE': 'school_type_code',
-                            'RELDENOM': 'religious_denomination',
-                            'AGERANGE': 'age_range',
-                            'TOTPUPS': 'total_pupils',
-                            'TELIG': 'eligible_pupils',
-                            # Core KS2 metrics
-                            'PTRWM_EXP': 'rwm_expected_pct',
-                            'PTRWM_HIGH': 'rwm_high_pct',
-                            'READPROG': 'reading_progress',
-                            'WRITPROG': 'writing_progress',
-                            'MATPROG': 'maths_progress',
-                            'PTREAD_EXP': 'reading_expected_pct',
-                            'PTWRITTA_EXP': 'writing_expected_pct',
-                            'PTMAT_EXP': 'maths_expected_pct',
-                            'READ_AVERAGE': 'reading_avg_score',
-                            'MAT_AVERAGE': 'maths_avg_score',
-                            'PTREAD_HIGH': 'reading_high_pct',
-                            'PTWRITTA_HIGH': 'writing_high_pct',
-                            'PTMAT_HIGH': 'maths_high_pct',
-                            # GPS (Grammar, Punctuation & Spelling)
-                            'PTGPS_EXP': 'gps_expected_pct',
-                            'PTGPS_HIGH': 'gps_high_pct',
-                            'GPS_AVERAGE': 'gps_avg_score',
-                            # Science
-                            'PTSCITA_EXP': 'science_expected_pct',
-                            # School context
-                            'PTFSM6CLA1A': 'disadvantaged_pct',
-                            'PTEALGRP2': 'eal_pct',
-                            'PSENELK': 'sen_support_pct',
-                            'PSENELE': 'sen_ehcp_pct',
-                            'PTMOBN': 'stability_pct',
-                            # Gender breakdown
-                            'PTRWM_EXP_B': 'rwm_expected_boys_pct',
-                            'PTRWM_EXP_G': 'rwm_expected_girls_pct',
-                            'PTRWM_HIGH_B': 'rwm_high_boys_pct',
-                            'PTRWM_HIGH_G': 'rwm_high_girls_pct',
-                            # Disadvantaged performance
-                            'PTRWM_EXP_FSM6CLA1A': 'rwm_expected_disadvantaged_pct',
-                            'PTRWM_EXP_NotFSM6CLA1A': 'rwm_expected_non_disadvantaged_pct',
-                            'DIFFN_RWM_EXP': 'disadvantaged_gap',
-                            # 3-year averages
-                            'PTRWM_EXP_3YR': 'rwm_expected_3yr_pct',
-                            'READ_AVERAGE_3YR': 'reading_avg_3yr',
-                            'MAT_AVERAGE_3YR': 'maths_avg_3yr',
-                        })
-                        
-                        # Create address field
-                        def make_address(row):
-                            parts = []
-                            if pd.notna(row.get('address1')) and row.get('address1'):
-                                parts.append(str(row['address1']))
-                            if pd.notna(row.get('town')) and row.get('town'):
-                                parts.append(str(row['town']))
-                            if pd.notna(row.get('postcode')) and row.get('postcode'):
-                                parts.append(str(row['postcode']))
-                            return ', '.join(parts) if parts else ''
-                        
-                        df['address'] = df.apply(make_address, axis=1)
-                        
-                        # Map school type codes to names
-                        school_type_map = {
-                            'AC': 'Academy', 'ACC': 'Academy Converter', 'ACS': 'Academy Sponsor Led',
-                            'CY': 'Community School', 'VA': 'Voluntary Aided', 'VC': 'Voluntary Controlled',
-                            'FD': 'Foundation', 'F': 'Foundation', 'FS': 'Free School',
-                        }
-                        df['school_type'] = df['school_type_code'].map(school_type_map).fillna('Other')
-                        
-                        # Parse numeric columns
-                        numeric_cols = [
-                            # Core metrics
-                            'rwm_expected_pct', 'rwm_high_pct', 'reading_progress', 
-                            'writing_progress', 'maths_progress', 'reading_expected_pct',
-                            'writing_expected_pct', 'maths_expected_pct', 'reading_avg_score',
-                            'maths_avg_score', 'reading_high_pct', 'writing_high_pct', 'maths_high_pct',
-                            # GPS & Science
-                            'gps_expected_pct', 'gps_high_pct', 'gps_avg_score', 'science_expected_pct',
-                            # School context
-                            'total_pupils', 'eligible_pupils', 'disadvantaged_pct', 'eal_pct',
-                            'sen_support_pct', 'sen_ehcp_pct', 'stability_pct',
-                            # Gender breakdown
-                            'rwm_expected_boys_pct', 'rwm_expected_girls_pct',
-                            'rwm_high_boys_pct', 'rwm_high_girls_pct',
-                            # Disadvantaged performance
-                            'rwm_expected_disadvantaged_pct', 'rwm_expected_non_disadvantaged_pct', 'disadvantaged_gap',
-                            # 3-year averages
-                            'rwm_expected_3yr_pct', 'reading_avg_3yr', 'maths_avg_3yr',
-                        ]
-                        
-                        for col in numeric_cols:
-                            if col in df.columns:
-                                df[col] = df[col].apply(parse_numeric)
-                        
-                        all_data.append(df)
-                        print(f"  Loaded {len(df)} schools for year {year}")
-                        
-                    except Exception as e:
-                        print(f"Error loading {ks2_file}: {e}")
-    
-    if all_data:
-        _data_cache = pd.concat(all_data, ignore_index=True)
-        print(f"\nTotal records loaded: {len(_data_cache)}")
-        print(f"Unique schools: {_data_cache['urn'].nunique()}")
-        print(f"Years: {sorted(_data_cache['year'].unique())}")
-    else:
-        print("No data files found. Creating empty DataFrame.")
-        _data_cache = pd.DataFrame()
-    
-    return _data_cache
-
 
 @app.get("/")
 async def root():
     """Serve the frontend."""
-    return FileResponse(FRONTEND_DIR / "index.html")
+    return FileResponse(settings.frontend_dir / "index.html")
 
 
 @app.get("/api/schools")
 async def get_schools(
     search: Optional[str] = Query(None, description="Search by school name"),
-    local_authority: Optional[str] = Query(None, description="Filter by local authority (Wandsworth or Merton)"),
+    local_authority: Optional[str] = Query(None, description="Filter by local authority"),
     school_type: Optional[str] = Query(None, description="Filter by school type"),
+    page: int = Query(1, ge=1, description="Page number"),
+    page_size: int = Query(None, ge=1, le=100, description="Results per page"),
 ):
-    """Get list of unique primary schools in Wandsworth and Merton."""
+    """
+    Get list of unique primary schools with pagination.
+    
+    Returns paginated results with total count for efficient loading.
+    """
     df = load_school_data()
     
     if df.empty:
-        return {"schools": []}
+        return {"schools": [], "total": 0, "page": page, "page_size": 0}
+    
+    # Use configured default if not specified
+    if page_size is None:
+        page_size = settings.default_page_size
     
     # Get unique schools (latest year data for each)
     latest_year = df.groupby('urn')['year'].max().reset_index()
     df_latest = df.merge(latest_year, on=['urn', 'year'])
     
-    school_cols = ["urn", "school_name", "local_authority", "school_type", "address", "town", "postcode"]
+    available_cols = [c for c in SCHOOL_COLUMNS if c in df_latest.columns]
-    available_cols = [c for c in school_cols if c in df_latest.columns]
     schools_df = df_latest[available_cols].drop_duplicates(subset=['urn'])
     
     # Apply filters
@@ -298,7 +97,19 @@ async def get_schools(
     if school_type:
         schools_df = schools_df[schools_df["school_type"].str.lower() == school_type.lower()]
     
-    return {"schools": clean_for_json(schools_df)}
+    # Pagination
+    total = len(schools_df)
+    start_idx = (page - 1) * page_size
+    end_idx = start_idx + page_size
+    schools_df = schools_df.iloc[start_idx:end_idx]
+    
+    return {
+        "schools": clean_for_json(schools_df),
+        "total": total,
+        "page": page,
+        "page_size": page_size,
+        "total_pages": (total + page_size - 1) // page_size if page_size > 0 else 0,
+    }
 
 
 @app.get("/api/schools/{urn}")
@@ -390,56 +201,18 @@ async def get_filter_options():
 
 @app.get("/api/metrics")
 async def get_available_metrics():
-    """Get list of available KS2 performance metrics for primary schools."""
+    """
+    Get list of available KS2 performance metrics for primary schools.
+    
+    This is the single source of truth for metric definitions.
+    Frontend should consume this to avoid duplication.
+    """
     df = load_school_data()
     
-    # Define KS2 metric metadata organized by category
-    metric_info = {
-        # Expected Standard
-        "rwm_expected_pct": {"name": "RWM Combined %", "description": "% meeting expected standard in reading, writing and maths", "type": "percentage", "category": "expected"},
-        "reading_expected_pct": {"name": "Reading Expected %", "description": "% meeting expected standard in reading", "type": "percentage", "category": "expected"},
-        "writing_expected_pct": {"name": "Writing Expected %", "description": "% meeting expected standard in writing", "type": "percentage", "category": "expected"},
-        "maths_expected_pct": {"name": "Maths Expected %", "description": "% meeting expected standard in maths", "type": "percentage", "category": "expected"},
-        "gps_expected_pct": {"name": "GPS Expected %", "description": "% meeting expected standard in grammar, punctuation & spelling", "type": "percentage", "category": "expected"},
-        "science_expected_pct": {"name": "Science Expected %", "description": "% meeting expected standard in science", "type": "percentage", "category": "expected"},
-        # Higher Standard
-        "rwm_high_pct": {"name": "RWM Combined Higher %", "description": "% achieving higher standard in RWM combined", "type": "percentage", "category": "higher"},
-        "reading_high_pct": {"name": "Reading Higher %", "description": "% achieving higher standard in reading", "type": "percentage", "category": "higher"},
-        "writing_high_pct": {"name": "Writing Higher %", "description": "% achieving greater depth in writing", "type": "percentage", "category": "higher"},
-        "maths_high_pct": {"name": "Maths Higher %", "description": "% achieving higher standard in maths", "type": "percentage", "category": "higher"},
-        "gps_high_pct": {"name": "GPS Higher %", "description": "% achieving higher standard in GPS", "type": "percentage", "category": "higher"},
-        # Progress Scores
-        "reading_progress": {"name": "Reading Progress", "description": "Progress in reading from KS1 to KS2", "type": "score", "category": "progress"},
-        "writing_progress": {"name": "Writing Progress", "description": "Progress in writing from KS1 to KS2", "type": "score", "category": "progress"},
-        "maths_progress": {"name": "Maths Progress", "description": "Progress in maths from KS1 to KS2", "type": "score", "category": "progress"},
-        # Average Scores
-        "reading_avg_score": {"name": "Reading Avg Score", "description": "Average scaled score in reading", "type": "score", "category": "average"},
-        "maths_avg_score": {"name": "Maths Avg Score", "description": "Average scaled score in maths", "type": "score", "category": "average"},
-        "gps_avg_score": {"name": "GPS Avg Score", "description": "Average scaled score in GPS", "type": "score", "category": "average"},
-        # Gender Performance
-        "rwm_expected_boys_pct": {"name": "RWM Expected % (Boys)", "description": "% of boys meeting expected standard", "type": "percentage", "category": "gender"},
-        "rwm_expected_girls_pct": {"name": "RWM Expected % (Girls)", "description": "% of girls meeting expected standard", "type": "percentage", "category": "gender"},
-        "rwm_high_boys_pct": {"name": "RWM Higher % (Boys)", "description": "% of boys at higher standard", "type": "percentage", "category": "gender"},
-        "rwm_high_girls_pct": {"name": "RWM Higher % (Girls)", "description": "% of girls at higher standard", "type": "percentage", "category": "gender"},
-        # Disadvantaged Performance
-        "rwm_expected_disadvantaged_pct": {"name": "RWM Expected % (Disadvantaged)", "description": "% of disadvantaged pupils meeting expected", "type": "percentage", "category": "equity"},
-        "rwm_expected_non_disadvantaged_pct": {"name": "RWM Expected % (Non-Disadvantaged)", "description": "% of non-disadvantaged pupils meeting expected", "type": "percentage", "category": "equity"},
-        "disadvantaged_gap": {"name": "Disadvantaged Gap", "description": "Gap between disadvantaged and national non-disadvantaged", "type": "score", "category": "equity"},
-        # School Context
-        "disadvantaged_pct": {"name": "% Disadvantaged Pupils", "description": "% of pupils eligible for free school meals or looked after", "type": "percentage", "category": "context"},
-        "eal_pct": {"name": "% EAL Pupils", "description": "% of pupils with English as additional language", "type": "percentage", "category": "context"},
-        "sen_support_pct": {"name": "% SEN Support", "description": "% of pupils with SEN support", "type": "percentage", "category": "context"},
-        "stability_pct": {"name": "% Pupil Stability", "description": "% of non-mobile pupils (stayed at school)", "type": "percentage", "category": "context"},
-        # 3-Year Averages
-        "rwm_expected_3yr_pct": {"name": "RWM Expected % (3-Year Avg)", "description": "3-year average % meeting expected", "type": "percentage", "category": "trends"},
-        "reading_avg_3yr": {"name": "Reading Score (3-Year Avg)", "description": "3-year average reading score", "type": "score", "category": "trends"},
-        "maths_avg_3yr": {"name": "Maths Score (3-Year Avg)", "description": "3-year average maths score", "type": "score", "category": "trends"},
-    }
-    
     available = []
-    for col, info in metric_info.items():
+    for key, info in METRIC_DEFINITIONS.items():
-        if df.empty or col in df.columns:
+        if df.empty or key in df.columns:
-            available.append({"key": col, **info})
+            available.append({"key": key, **info})
     
     return {"metrics": available}
 
@@ -448,13 +221,14 @@ async def get_available_metrics():
 async def get_rankings(
     metric: str = Query("rwm_expected_pct", description="KS2 metric to rank by"),
     year: Optional[int] = Query(None, description="Specific year (defaults to most recent)"),
-    limit: int = Query(20, description="Number of schools to return"),
+    limit: int = Query(20, ge=1, le=100, description="Number of schools to return"),
+    local_authority: Optional[str] = Query(None, description="Filter by local authority"),
 ):
     """Get primary school rankings by a specific KS2 metric."""
     df = load_school_data()
     
     if df.empty:
-        return {"metric": metric, "year": None, "rankings": []}
+        return {"metric": metric, "year": None, "rankings": [], "total": 0}
     
     if metric not in df.columns:
         raise HTTPException(status_code=400, detail=f"Metric '{metric}' not available")
@@ -467,39 +241,26 @@ async def get_rankings(
         max_year = df["year"].max()
         df = df[df["year"] == max_year]
     
+    # Filter by local authority if specified
+    if local_authority:
+        df = df[df["local_authority"].str.lower() == local_authority.lower()]
+    
     # Sort and rank (exclude rows with no data for this metric)
     df = df.dropna(subset=[metric])
+    total = len(df)
     
     # For progress scores, higher is better. For percentages, higher is also better.
     df = df.sort_values(metric, ascending=False).head(limit)
     
     # Return only relevant fields for rankings
-    ranking_cols = [
+    available_cols = [c for c in RANKING_COLUMNS if c in df.columns]
-        "urn", "school_name", "local_authority", "school_type", "address", "year", "total_pupils",
-        # Core expected
-        "rwm_expected_pct", "reading_expected_pct", "writing_expected_pct", "maths_expected_pct",
-        "gps_expected_pct", "science_expected_pct",
-        # Core higher
-        "rwm_high_pct", "reading_high_pct", "writing_high_pct", "maths_high_pct", "gps_high_pct",
-        # Progress & averages
-        "reading_progress", "writing_progress", "maths_progress",
-        "reading_avg_score", "maths_avg_score", "gps_avg_score",
-        # Gender
-        "rwm_expected_boys_pct", "rwm_expected_girls_pct", "rwm_high_boys_pct", "rwm_high_girls_pct",
-        # Equity
-        "rwm_expected_disadvantaged_pct", "rwm_expected_non_disadvantaged_pct", "disadvantaged_gap",
-        # Context
-        "disadvantaged_pct", "eal_pct", "sen_support_pct", "stability_pct",
-        # 3-year
-        "rwm_expected_3yr_pct", "reading_avg_3yr", "maths_avg_3yr",
-    ]
-    available_cols = [c for c in ranking_cols if c in df.columns]
     df = df[available_cols]
     
     return {
         "metric": metric,
         "year": int(df["year"].iloc[0]) if not df.empty else None,
-        "rankings": clean_for_json(df)
+        "rankings": clean_for_json(df),
+        "total": total,
     }
 
 
@@ -512,7 +273,7 @@ async def get_data_info():
         return {
             "status": "no_data",
             "message": "No data files found in data folder. Please download KS2 data from the government website.",
-            "data_folder": str(DATA_DIR),
+            "data_folder": str(settings.data_dir),
         }
     
     years = [int(y) for y in sorted(df["year"].unique())]
@@ -529,17 +290,22 @@ async def get_data_info():
     }
 
 
-# Mount static files
+@app.post("/api/admin/reload")
-@app.on_event("startup")
+async def reload_data():
-async def startup():
+    """Admin endpoint to force data reload (useful after data updates)."""
-    """Setup static file serving and load data on startup."""
+    clear_cache()
-    if FRONTEND_DIR.exists():
-        app.mount("/static", StaticFiles(directory=FRONTEND_DIR), name="static")
-    
-    # Pre-load data
     load_school_data()
+    return {"status": "reloaded"}
+
+
+# Mount static files after all routes are defined
+@app.on_event("startup")
+async def mount_static():
+    """Mount static file serving."""
+    if settings.frontend_dir.exists():
+        app.mount("/static", StaticFiles(directory=settings.frontend_dir), name="static")
 
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    uvicorn.run(app, host=settings.host, port=settings.port)

backend/config.py

@@ -0,0 +1,38 @@
+"""
+Application configuration using pydantic-settings.
+Loads from environment variables and .env file.
+"""
+
+from pathlib import Path
+from typing import List
+from pydantic_settings import BaseSettings
+import os
+
+
+class Settings(BaseSettings):
+    """Application settings loaded from environment."""
+    
+    # Paths
+    data_dir: Path = Path(__file__).parent.parent / "data"
+    frontend_dir: Path = Path(__file__).parent.parent / "frontend"
+    
+    # Server
+    host: str = "0.0.0.0"
+    port: int = 80
+    
+    # CORS
+    allowed_origins: List[str] = ["https://schoolcompare.co.uk", "http://localhost:8000", "http://localhost:3000"]
+    
+    # API
+    default_page_size: int = 50
+    max_page_size: int = 100
+    
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+        extra = "ignore"
+
+
+# Singleton instance
+settings = Settings()
+

backend/data_loader.py

@@ -0,0 +1,196 @@
+"""
+Data loading module with optimized pandas operations.
+Uses vectorized operations instead of .apply() for performance.
+"""
+
+import pandas as pd
+import numpy as np
+from pathlib import Path
+from functools import lru_cache
+import re
+from typing import Optional
+
+from .config import settings
+from .schemas import (
+    COLUMN_MAPPINGS,
+    NUMERIC_COLUMNS,
+    SCHOOL_TYPE_MAP,
+    NULL_VALUES,
+    LA_CODE_TO_NAME,
+)
+
+
+def extract_year_from_folder(folder_name: str) -> Optional[int]:
+    """Extract the end year from folder name like '2023-2024' -> 2024."""
+    match = re.search(r'(\d{4})-(\d{4})', folder_name)
+    if match:
+        return int(match.group(2))
+    return None
+
+
+def parse_numeric_vectorized(series: pd.Series) -> pd.Series:
+    """
+    Vectorized numeric parsing - much faster than .apply().
+    Handles SUPP, NE, NA, NP, %, etc.
+    """
+    # Convert to string first
+    str_series = series.astype(str)
+    
+    # Replace null values with NaN
+    for null_val in NULL_VALUES:
+        str_series = str_series.replace(null_val, np.nan)
+    
+    # Remove % signs
+    str_series = str_series.str.rstrip('%')
+    
+    # Convert to numeric
+    return pd.to_numeric(str_series, errors='coerce')
+
+
+def create_address_vectorized(df: pd.DataFrame) -> pd.Series:
+    """
+    Vectorized address creation - much faster than .apply().
+    """
+    parts = []
+    
+    if 'address1' in df.columns:
+        parts.append(df['address1'].fillna('').astype(str))
+    if 'town' in df.columns:
+        parts.append(df['town'].fillna('').astype(str))
+    if 'postcode' in df.columns:
+        parts.append(df['postcode'].fillna('').astype(str))
+    
+    if not parts:
+        return pd.Series([''] * len(df), index=df.index)
+    
+    # Combine parts with comma separator, filtering empty strings
+    result = pd.Series([''] * len(df), index=df.index)
+    for i, row_idx in enumerate(df.index):
+        row_parts = [p.iloc[i] if hasattr(p, 'iloc') else p[i] for p in parts]
+        row_parts = [p for p in row_parts if p and p.strip()]
+        result.iloc[i] = ', '.join(row_parts)
+    
+    return result
+
+
+def create_address_fast(df: pd.DataFrame) -> pd.Series:
+    """
+    Fast vectorized address creation using string concatenation.
+    """
+    addr1 = df.get('address1', pd.Series([''] * len(df))).fillna('').astype(str)
+    town = df.get('town', pd.Series([''] * len(df))).fillna('').astype(str)
+    postcode = df.get('postcode', pd.Series([''] * len(df))).fillna('').astype(str)
+    
+    # Build address with proper separators
+    result = addr1.str.strip()
+    
+    # Add town if not empty
+    town_mask = town.str.strip() != ''
+    result = result.where(~town_mask, result + ', ' + town.str.strip())
+    
+    # Add postcode if not empty  
+    postcode_mask = postcode.str.strip() != ''
+    result = result.where(~postcode_mask, result + ', ' + postcode.str.strip())
+    
+    # Clean up leading commas
+    result = result.str.lstrip(', ')
+    
+    return result
+
+
+def load_year_data(year_folder: Path, year: int) -> Optional[pd.DataFrame]:
+    """Load and process data for a single year."""
+    ks2_file = year_folder / "england_ks2final.csv"
+    if not ks2_file.exists():
+        return None
+    
+    try:
+        print(f"Loading data from {ks2_file}")
+        df = pd.read_csv(ks2_file, low_memory=False)
+        
+        # Handle column types
+        if 'LEA' in df.columns and df['LEA'].dtype == 'object':
+            df['LEA'] = pd.to_numeric(df['LEA'], errors='coerce')
+        if 'URN' in df.columns and df['URN'].dtype == 'object':
+            df['URN'] = pd.to_numeric(df['URN'], errors='coerce')
+        
+        # Filter to schools only (RECTYPE == 1 means school level data)
+        if 'RECTYPE' in df.columns:
+            df = df[df['RECTYPE'] == 1].copy()
+        
+        # Add year and local authority name
+        df['year'] = year
+        
+        # Try different column names for LA name
+        la_name_cols = ['LANAME', 'LA (name)', 'LA_NAME', 'LA NAME']
+        la_col_found = None
+        for col in la_name_cols:
+            if col in df.columns:
+                la_col_found = col
+                break
+        
+        if la_col_found:
+            df['local_authority'] = df[la_col_found]
+        elif 'LEA' in df.columns:
+            # Map LEA codes to names using our mapping
+            df['local_authority'] = df['LEA'].map(LA_CODE_TO_NAME).fillna(df['LEA'].astype(str))
+        
+        # Rename columns using mapping
+        rename_dict = {k: v for k, v in COLUMN_MAPPINGS.items() if k in df.columns}
+        df = df.rename(columns=rename_dict)
+        
+        # Create address field (vectorized)
+        df['address'] = create_address_fast(df)
+        
+        # Map school type codes to names (vectorized)
+        if 'school_type_code' in df.columns:
+            df['school_type'] = df['school_type_code'].map(SCHOOL_TYPE_MAP).fillna('Other')
+        
+        # Parse numeric columns (vectorized - much faster than .apply())
+        for col in NUMERIC_COLUMNS:
+            if col in df.columns:
+                df[col] = parse_numeric_vectorized(df[col])
+        
+        print(f"  Loaded {len(df)} schools for year {year}")
+        return df
+        
+    except Exception as e:
+        print(f"Error loading {ks2_file}: {e}")
+        return None
+
+
+@lru_cache(maxsize=1)
+def load_school_data() -> pd.DataFrame:
+    """
+    Load and combine all school data from CSV files in year folders.
+    Uses lru_cache for singleton-like behavior.
+    """
+    all_data = []
+    
+    data_dir = settings.data_dir
+    if data_dir.exists():
+        for year_folder in data_dir.iterdir():
+            if year_folder.is_dir() and re.match(r'\d{4}-\d{4}', year_folder.name):
+                year = extract_year_from_folder(year_folder.name)
+                if year is None:
+                    continue
+                
+                df = load_year_data(year_folder, year)
+                if df is not None:
+                    all_data.append(df)
+    
+    if all_data:
+        result = pd.concat(all_data, ignore_index=True)
+        print(f"\nTotal records loaded: {len(result)}")
+        print(f"Unique schools: {result['urn'].nunique()}")
+        print(f"Years: {sorted(result['year'].unique())}")
+        return result
+    else:
+        print("No data files found. Creating empty DataFrame.")
+        return pd.DataFrame()
+
+
+def clear_cache():
+    """Clear the data cache to force reload."""
+    load_school_data.cache_clear()
+

backend/schemas.py

@@ -0,0 +1,398 @@
+"""
+Schema definitions: column mappings, metric definitions, school type mappings.
+Single source of truth for all data transformations.
+"""
+
+# Column name mappings from DfE CSV to API field names
+COLUMN_MAPPINGS = {
+    'URN': 'urn',
+    'SCHNAME': 'school_name',
+    'ADDRESS1': 'address1',
+    'ADDRESS2': 'address2',
+    'TOWN': 'town',
+    'PCODE': 'postcode',
+    'NFTYPE': 'school_type_code',
+    'RELDENOM': 'religious_denomination',
+    'AGERANGE': 'age_range',
+    'TOTPUPS': 'total_pupils',
+    'TELIG': 'eligible_pupils',
+    # Core KS2 metrics
+    'PTRWM_EXP': 'rwm_expected_pct',
+    'PTRWM_HIGH': 'rwm_high_pct',
+    'READPROG': 'reading_progress',
+    'WRITPROG': 'writing_progress',
+    'MATPROG': 'maths_progress',
+    'PTREAD_EXP': 'reading_expected_pct',
+    'PTWRITTA_EXP': 'writing_expected_pct',
+    'PTMAT_EXP': 'maths_expected_pct',
+    'READ_AVERAGE': 'reading_avg_score',
+    'MAT_AVERAGE': 'maths_avg_score',
+    'PTREAD_HIGH': 'reading_high_pct',
+    'PTWRITTA_HIGH': 'writing_high_pct',
+    'PTMAT_HIGH': 'maths_high_pct',
+    # GPS (Grammar, Punctuation & Spelling)
+    'PTGPS_EXP': 'gps_expected_pct',
+    'PTGPS_HIGH': 'gps_high_pct',
+    'GPS_AVERAGE': 'gps_avg_score',
+    # Science
+    'PTSCITA_EXP': 'science_expected_pct',
+    # School context
+    'PTFSM6CLA1A': 'disadvantaged_pct',
+    'PTEALGRP2': 'eal_pct',
+    'PSENELK': 'sen_support_pct',
+    'PSENELE': 'sen_ehcp_pct',
+    'PTMOBN': 'stability_pct',
+    # Gender breakdown
+    'PTRWM_EXP_B': 'rwm_expected_boys_pct',
+    'PTRWM_EXP_G': 'rwm_expected_girls_pct',
+    'PTRWM_HIGH_B': 'rwm_high_boys_pct',
+    'PTRWM_HIGH_G': 'rwm_high_girls_pct',
+    # Disadvantaged performance
+    'PTRWM_EXP_FSM6CLA1A': 'rwm_expected_disadvantaged_pct',
+    'PTRWM_EXP_NotFSM6CLA1A': 'rwm_expected_non_disadvantaged_pct',
+    'DIFFN_RWM_EXP': 'disadvantaged_gap',
+    # 3-year averages
+    'PTRWM_EXP_3YR': 'rwm_expected_3yr_pct',
+    'READ_AVERAGE_3YR': 'reading_avg_3yr',
+    'MAT_AVERAGE_3YR': 'maths_avg_3yr',
+}
+
+# Numeric columns that need parsing
+NUMERIC_COLUMNS = [
+    # Core metrics
+    'rwm_expected_pct', 'rwm_high_pct', 'reading_progress', 
+    'writing_progress', 'maths_progress', 'reading_expected_pct',
+    'writing_expected_pct', 'maths_expected_pct', 'reading_avg_score',
+    'maths_avg_score', 'reading_high_pct', 'writing_high_pct', 'maths_high_pct',
+    # GPS & Science
+    'gps_expected_pct', 'gps_high_pct', 'gps_avg_score', 'science_expected_pct',
+    # School context
+    'total_pupils', 'eligible_pupils', 'disadvantaged_pct', 'eal_pct',
+    'sen_support_pct', 'sen_ehcp_pct', 'stability_pct',
+    # Gender breakdown
+    'rwm_expected_boys_pct', 'rwm_expected_girls_pct',
+    'rwm_high_boys_pct', 'rwm_high_girls_pct',
+    # Disadvantaged performance
+    'rwm_expected_disadvantaged_pct', 'rwm_expected_non_disadvantaged_pct', 'disadvantaged_gap',
+    # 3-year averages
+    'rwm_expected_3yr_pct', 'reading_avg_3yr', 'maths_avg_3yr',
+]
+
+# School type code to name mapping
+SCHOOL_TYPE_MAP = {
+    'AC': 'Academy',
+    'ACC': 'Academy Converter',
+    'ACS': 'Academy Sponsor Led',
+    'CY': 'Community School',
+    'VA': 'Voluntary Aided',
+    'VC': 'Voluntary Controlled',
+    'FD': 'Foundation',
+    'F': 'Foundation',
+    'FS': 'Free School',
+}
+
+# Special values to treat as null
+NULL_VALUES = ['SUPP', 'NE', 'NA', 'NP', 'NEW', 'LOW', '']
+
+# KS2 Metric definitions - single source of truth
+# Used by both backend API and frontend
+METRIC_DEFINITIONS = {
+    # Expected Standard
+    "rwm_expected_pct": {
+        "name": "RWM Combined %",
+        "short_name": "RWM %",
+        "description": "% meeting expected standard in reading, writing and maths",
+        "type": "percentage",
+        "category": "expected"
+    },
+    "reading_expected_pct": {
+        "name": "Reading Expected %",
+        "short_name": "Reading %",
+        "description": "% meeting expected standard in reading",
+        "type": "percentage",
+        "category": "expected"
+    },
+    "writing_expected_pct": {
+        "name": "Writing Expected %",
+        "short_name": "Writing %",
+        "description": "% meeting expected standard in writing",
+        "type": "percentage",
+        "category": "expected"
+    },
+    "maths_expected_pct": {
+        "name": "Maths Expected %",
+        "short_name": "Maths %",
+        "description": "% meeting expected standard in maths",
+        "type": "percentage",
+        "category": "expected"
+    },
+    "gps_expected_pct": {
+        "name": "GPS Expected %",
+        "short_name": "GPS %",
+        "description": "% meeting expected standard in grammar, punctuation & spelling",
+        "type": "percentage",
+        "category": "expected"
+    },
+    "science_expected_pct": {
+        "name": "Science Expected %",
+        "short_name": "Science %",
+        "description": "% meeting expected standard in science",
+        "type": "percentage",
+        "category": "expected"
+    },
+    # Higher Standard
+    "rwm_high_pct": {
+        "name": "RWM Combined Higher %",
+        "short_name": "RWM Higher %",
+        "description": "% achieving higher standard in RWM combined",
+        "type": "percentage",
+        "category": "higher"
+    },
+    "reading_high_pct": {
+        "name": "Reading Higher %",
+        "short_name": "Reading Higher %",
+        "description": "% achieving higher standard in reading",
+        "type": "percentage",
+        "category": "higher"
+    },
+    "writing_high_pct": {
+        "name": "Writing Higher %",
+        "short_name": "Writing Higher %",
+        "description": "% achieving greater depth in writing",
+        "type": "percentage",
+        "category": "higher"
+    },
+    "maths_high_pct": {
+        "name": "Maths Higher %",
+        "short_name": "Maths Higher %",
+        "description": "% achieving higher standard in maths",
+        "type": "percentage",
+        "category": "higher"
+    },
+    "gps_high_pct": {
+        "name": "GPS Higher %",
+        "short_name": "GPS Higher %",
+        "description": "% achieving higher standard in GPS",
+        "type": "percentage",
+        "category": "higher"
+    },
+    # Progress Scores
+    "reading_progress": {
+        "name": "Reading Progress",
+        "short_name": "Reading Progress",
+        "description": "Progress in reading from KS1 to KS2",
+        "type": "score",
+        "category": "progress"
+    },
+    "writing_progress": {
+        "name": "Writing Progress",
+        "short_name": "Writing Progress",
+        "description": "Progress in writing from KS1 to KS2",
+        "type": "score",
+        "category": "progress"
+    },
+    "maths_progress": {
+        "name": "Maths Progress",
+        "short_name": "Maths Progress",
+        "description": "Progress in maths from KS1 to KS2",
+        "type": "score",
+        "category": "progress"
+    },
+    # Average Scores
+    "reading_avg_score": {
+        "name": "Reading Average Score",
+        "short_name": "Reading Avg",
+        "description": "Average scaled score in reading",
+        "type": "score",
+        "category": "average"
+    },
+    "maths_avg_score": {
+        "name": "Maths Average Score",
+        "short_name": "Maths Avg",
+        "description": "Average scaled score in maths",
+        "type": "score",
+        "category": "average"
+    },
+    "gps_avg_score": {
+        "name": "GPS Average Score",
+        "short_name": "GPS Avg",
+        "description": "Average scaled score in GPS",
+        "type": "score",
+        "category": "average"
+    },
+    # Gender Performance
+    "rwm_expected_boys_pct": {
+        "name": "RWM Expected % (Boys)",
+        "short_name": "Boys RWM %",
+        "description": "% of boys meeting expected standard",
+        "type": "percentage",
+        "category": "gender"
+    },
+    "rwm_expected_girls_pct": {
+        "name": "RWM Expected % (Girls)",
+        "short_name": "Girls RWM %",
+        "description": "% of girls meeting expected standard",
+        "type": "percentage",
+        "category": "gender"
+    },
+    "rwm_high_boys_pct": {
+        "name": "RWM Higher % (Boys)",
+        "short_name": "Boys Higher %",
+        "description": "% of boys at higher standard",
+        "type": "percentage",
+        "category": "gender"
+    },
+    "rwm_high_girls_pct": {
+        "name": "RWM Higher % (Girls)",
+        "short_name": "Girls Higher %",
+        "description": "% of girls at higher standard",
+        "type": "percentage",
+        "category": "gender"
+    },
+    # Disadvantaged Performance
+    "rwm_expected_disadvantaged_pct": {
+        "name": "RWM Expected % (Disadvantaged)",
+        "short_name": "Disadvantaged %",
+        "description": "% of disadvantaged pupils meeting expected",
+        "type": "percentage",
+        "category": "equity"
+    },
+    "rwm_expected_non_disadvantaged_pct": {
+        "name": "RWM Expected % (Non-Disadvantaged)",
+        "short_name": "Non-Disadv %",
+        "description": "% of non-disadvantaged pupils meeting expected",
+        "type": "percentage",
+        "category": "equity"
+    },
+    "disadvantaged_gap": {
+        "name": "Disadvantaged Gap",
+        "short_name": "Disadv Gap",
+        "description": "Gap between disadvantaged and national non-disadvantaged",
+        "type": "score",
+        "category": "equity"
+    },
+    # School Context
+    "disadvantaged_pct": {
+        "name": "% Disadvantaged Pupils",
+        "short_name": "% Disadvantaged",
+        "description": "% of pupils eligible for free school meals or looked after",
+        "type": "percentage",
+        "category": "context"
+    },
+    "eal_pct": {
+        "name": "% EAL Pupils",
+        "short_name": "% EAL",
+        "description": "% of pupils with English as additional language",
+        "type": "percentage",
+        "category": "context"
+    },
+    "sen_support_pct": {
+        "name": "% SEN Support",
+        "short_name": "% SEN",
+        "description": "% of pupils with SEN support",
+        "type": "percentage",
+        "category": "context"
+    },
+    "stability_pct": {
+        "name": "% Pupil Stability",
+        "short_name": "% Stable",
+        "description": "% of non-mobile pupils (stayed at school)",
+        "type": "percentage",
+        "category": "context"
+    },
+    # 3-Year Averages
+    "rwm_expected_3yr_pct": {
+        "name": "RWM Expected % (3-Year Avg)",
+        "short_name": "RWM 3yr %",
+        "description": "3-year average % meeting expected",
+        "type": "percentage",
+        "category": "trends"
+    },
+    "reading_avg_3yr": {
+        "name": "Reading Score (3-Year Avg)",
+        "short_name": "Reading 3yr",
+        "description": "3-year average reading score",
+        "type": "score",
+        "category": "trends"
+    },
+    "maths_avg_3yr": {
+        "name": "Maths Score (3-Year Avg)",
+        "short_name": "Maths 3yr",
+        "description": "3-year average maths score",
+        "type": "score",
+        "category": "trends"
+    },
+}
+
+# Ranking columns to include in rankings response
+RANKING_COLUMNS = [
+    "urn", "school_name", "local_authority", "school_type", "address", "year", "total_pupils",
+    # Core expected
+    "rwm_expected_pct", "reading_expected_pct", "writing_expected_pct", "maths_expected_pct",
+    "gps_expected_pct", "science_expected_pct",
+    # Core higher
+    "rwm_high_pct", "reading_high_pct", "writing_high_pct", "maths_high_pct", "gps_high_pct",
+    # Progress & averages
+    "reading_progress", "writing_progress", "maths_progress",
+    "reading_avg_score", "maths_avg_score", "gps_avg_score",
+    # Gender
+    "rwm_expected_boys_pct", "rwm_expected_girls_pct", "rwm_high_boys_pct", "rwm_high_girls_pct",
+    # Equity
+    "rwm_expected_disadvantaged_pct", "rwm_expected_non_disadvantaged_pct", "disadvantaged_gap",
+    # Context
+    "disadvantaged_pct", "eal_pct", "sen_support_pct", "stability_pct",
+    # 3-year
+    "rwm_expected_3yr_pct", "reading_avg_3yr", "maths_avg_3yr",
+]
+
+# School listing columns
+SCHOOL_COLUMNS = ["urn", "school_name", "local_authority", "school_type", "address", "town", "postcode"]
+
+# Local Authority code to name mapping (for fallback when LANAME column missing)
+# Source: https://www.gov.uk/government/publications/local-authority-codes
+LA_CODE_TO_NAME = {
+    201: "City of London", 202: "Camden", 203: "Greenwich", 204: "Hackney",
+    205: "Hammersmith and Fulham", 206: "Islington", 207: "Kensington and Chelsea",
+    208: "Lambeth", 209: "Lewisham", 210: "Southwark", 211: "Tower Hamlets",
+    212: "Wandsworth", 213: "Westminster", 301: "Barking and Dagenham", 302: "Barnet",
+    303: "Bexley", 304: "Brent", 305: "Bromley", 306: "Croydon", 307: "Ealing",
+    308: "Enfield", 309: "Haringey", 310: "Harrow", 311: "Havering", 312: "Hillingdon",
+    313: "Hounslow", 314: "Kingston upon Thames", 315: "Merton", 316: "Newham",
+    317: "Redbridge", 318: "Richmond upon Thames", 319: "Sutton", 320: "Waltham Forest",
+    330: "Birmingham", 331: "Coventry", 332: "Dudley", 333: "Sandwell", 334: "Solihull",
+    335: "Walsall", 336: "Wolverhampton", 340: "Knowsley", 341: "Liverpool",
+    342: "St. Helens", 343: "Sefton", 344: "Wirral", 350: "Bolton", 351: "Bury",
+    352: "Manchester", 353: "Oldham", 354: "Rochdale", 355: "Salford", 356: "Stockport",
+    357: "Tameside", 358: "Trafford", 359: "Wigan", 370: "Barnsley", 371: "Doncaster",
+    372: "Rotherham", 373: "Sheffield", 380: "Bradford", 381: "Calderdale",
+    382: "Kirklees", 383: "Leeds", 384: "Wakefield", 390: "Gateshead",
+    391: "Newcastle upon Tyne", 392: "North Tyneside", 393: "South Tyneside",
+    394: "Sunderland", 420: "Isles of Scilly", 800: "Bath and North East Somerset",
+    801: "Bristol, City of", 802: "North Somerset", 803: "South Gloucestershire",
+    805: "Hartlepool", 806: "Middlesbrough", 807: "Redcar and Cleveland",
+    808: "Stockton-on-Tees", 810: "Kingston Upon Hull, City of", 811: "East Riding of Yorkshire",
+    812: "North East Lincolnshire", 813: "North Lincolnshire", 815: "North Yorkshire",
+    816: "York", 820: "Bedford", 821: "Central Bedfordshire", 822: "Luton",
+    825: "Buckinghamshire", 826: "Milton Keynes", 830: "Derbyshire", 831: "Derby",
+    835: "Dorset", 836: "Bournemouth, Christchurch and Poole", 837: "Poole",
+    838: "Bournemouth", 839: "Durham", 840: "Darlington", 841: "East Sussex",
+    845: "Brighton and Hove", 846: "Hampshire", 850: "Portsmouth", 851: "Southampton",
+    852: "Isle of Wight", 855: "Leicestershire", 856: "Leicester", 857: "Rutland",
+    860: "Staffordshire", 861: "Stoke-on-Trent", 865: "Wiltshire", 866: "Swindon",
+    867: "Bracknell Forest", 868: "Windsor and Maidenhead", 869: "West Berkshire",
+    870: "Reading", 871: "Slough", 872: "Wokingham", 873: "Cambridgeshire",
+    874: "Peterborough", 876: "Halton", 877: "Warrington", 878: "Devon",
+    879: "Plymouth", 880: "Torbay", 881: "Essex", 882: "Southend-on-Sea",
+    883: "Thurrock", 884: "Herefordshire", 885: "Worcestershire", 886: "Kent",
+    887: "Medway", 888: "Lancashire", 889: "Blackburn with Darwen", 890: "Blackpool",
+    891: "Nottinghamshire", 892: "Nottingham", 893: "Shropshire", 894: "Telford and Wrekin",
+    895: "Cheshire East", 896: "Cheshire West and Chester", 908: "Cornwall",
+    909: "Cumbria", 916: "Gloucestershire", 919: "Hertfordshire", 921: "Norfolk",
+    925: "Lincolnshire", 926: "Northamptonshire", 928: "Northumberland",
+    929: "Oxfordshire", 931: "Somerset", 933: "Suffolk", 935: "Surrey",
+    936: "Warwickshire", 937: "West Sussex", 938: "Westmorland and Furness",
+    940: "Cumberland",
+    # Additional codes
+    420: "Isles of Scilly",
+}
+

backend/utils.py

@@ -0,0 +1,37 @@
+"""
+Utility functions for data conversion and JSON serialization.
+"""
+
+import pandas as pd
+import numpy as np
+from typing import Any, List
+
+
+def convert_to_native(value: Any) -> Any:
+    """Convert numpy types to native Python types for JSON serialization."""
+    if pd.isna(value):
+        return None
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, (np.floating,)):
+        if np.isnan(value) or np.isinf(value):
+            return None
+        return float(value)
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    if value == "SUPP" or value == "NE" or value == "NA" or value == "NP":
+        return None
+    return value
+
+
+def clean_for_json(df: pd.DataFrame) -> List[dict]:
+    """Convert DataFrame to list of dicts, replacing NaN/inf with None for JSON serialization."""
+    records = df.to_dict(orient="records")
+    cleaned = []
+    for record in records:
+        clean_record = {}
+        for key, value in record.items():
+            clean_record[key] = convert_to_native(value)
+        cleaned.append(clean_record)
+    return cleaned
+

frontend/app.js

@@ -5,12 +5,30 @@
 
 const API_BASE = '';
 
-// State
+// =============================================================================
-let allSchools = [];
+// STATE MANAGEMENT
-let selectedSchools = [];
+// =============================================================================
+
+const state = {
+    schools: [],
+    selectedSchools: [],
+    currentSchoolData: null,
+    filters: null,  // Cached filter data
+    metrics: null,  // Cached metric definitions
+    pagination: { page: 1, pageSize: 50, total: 0, totalPages: 0 },
+    isShowingFeatured: true,  // Whether showing featured schools vs search results
+    loading: {
+        schools: false,
+        filters: false,
+        rankings: false,
+        comparison: false,
+        modal: false,
+    },
+};
+
+// Charts
 let comparisonChart = null;
 let schoolDetailChart = null;
-let currentSchoolData = null;
 
 // Chart colors
 const CHART_COLORS = [
@@ -24,7 +42,10 @@ const CHART_COLORS = [
     '#9b59b6',  // violet
 ];
 
-// DOM Elements
+// =============================================================================
+// DOM ELEMENTS
+// =============================================================================
+
 const elements = {
     schoolSearch: document.getElementById('school-search'),
     localAuthorityFilter: document.getElementById('local-authority-filter'),
@@ -51,33 +72,129 @@ const elements = {
     addToCompare: document.getElementById('add-to-compare'),
 };
 
-// Initialize
+// =============================================================================
-document.addEventListener('DOMContentLoaded', init);
+// API & CACHING
+// =============================================================================
 
-async function init() {
+const apiCache = new Map();
-    await loadFilters();
+const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
-    await loadSchools();
-    await loadRankingYears();
-    await loadRankings();
-    setupEventListeners();
-}
 
-// API Functions
+async function fetchAPI(endpoint, options = {}) {
-async function fetchAPI(endpoint) {
+    const { useCache = true, showLoading = null } = options;
+    const cacheKey = endpoint;
+    
+    // Check cache
+    if (useCache && apiCache.has(cacheKey)) {
+        const cached = apiCache.get(cacheKey);
+        if (Date.now() - cached.timestamp < CACHE_TTL) {
+            return cached.data;
+        }
+        apiCache.delete(cacheKey);
+    }
+    
+    // Set loading state
+    if (showLoading) {
+        state.loading[showLoading] = true;
+        updateLoadingUI(showLoading);
+    }
+    
     try {
         const response = await fetch(`${API_BASE}${endpoint}`);
         if (!response.ok) throw new Error(`HTTP ${response.status}`);
-        return await response.json();
+        const data = await response.json();
+        
+        // Cache the response
+        if (useCache) {
+            apiCache.set(cacheKey, { data, timestamp: Date.now() });
+        }
+        
+        return data;
     } catch (error) {
         console.error(`API Error (${endpoint}):`, error);
         return null;
+    } finally {
+        if (showLoading) {
+            state.loading[showLoading] = false;
+            updateLoadingUI(showLoading);
+        }
     }
 }
 
-async function loadFilters() {
+function updateLoadingUI(section) {
-    const data = await fetchAPI('/api/filters');
+    // Update UI based on loading state
-    if (!data) return;
+    switch (section) {
+        case 'schools':
+            if (state.loading.schools) {
+                elements.schoolsGrid.innerHTML = renderLoadingSkeleton(6);
+            }
+            break;
+        case 'rankings':
+            if (state.loading.rankings) {
+                elements.rankingsList.innerHTML = renderLoadingSkeleton(5, 'ranking');
+            }
+            break;
+        case 'modal':
+            if (state.loading.modal) {
+                elements.modalStats.innerHTML = '<div class="loading"><div class="loading-spinner"></div><p>Loading school data...</p></div>';
+            }
+            break;
+    }
+}
 
+function renderLoadingSkeleton(count, type = 'card') {
+    if (type === 'ranking') {
+        return Array(count).fill(0).map(() => `
+            <div class="ranking-item skeleton">
+                <div class="skeleton-circle"></div>
+                <div class="skeleton-content">
+                    <div class="skeleton-line" style="width: 60%"></div>
+                    <div class="skeleton-line short" style="width: 30%"></div>
+                </div>
+                <div class="skeleton-score"></div>
+            </div>
+        `).join('');
+    }
+    return Array(count).fill(0).map(() => `
+        <div class="school-card skeleton">
+            <div class="skeleton-line" style="width: 70%"></div>
+            <div class="skeleton-line short" style="width: 40%"></div>
+            <div class="skeleton-line" style="width: 90%"></div>
+        </div>
+    `).join('');
+}
+
+// =============================================================================
+// INITIALIZATION
+// =============================================================================
+
+document.addEventListener('DOMContentLoaded', init);
+
+async function init() {
+    // Load filters and metrics in parallel (single request for filters)
+    const [filtersData, metricsData] = await Promise.all([
+        fetchAPI('/api/filters', { showLoading: 'filters' }),
+        fetchAPI('/api/metrics'),
+    ]);
+    
+    // Cache and apply filters
+    if (filtersData) {
+        state.filters = filtersData;
+        populateFilters(filtersData);
+    }
+    
+    // Cache metrics
+    if (metricsData) {
+        state.metrics = metricsData.metrics;
+    }
+    
+    // Load initial data
+    await loadSchools();
+    await loadRankings();
+    
+    setupEventListeners();
+}
+
+function populateFilters(data) {
     // Populate local authority filter
     data.local_authorities.forEach(la => {
         const option = document.createElement('option');
@@ -93,50 +210,8 @@ async function loadFilters() {
         option.textContent = type;
         elements.typeFilter.appendChild(option);
     });
-}
-
-async function loadSchools() {
-    const params = new URLSearchParams();
-    
-    const search = elements.schoolSearch.value.trim();
-    if (search) params.append('search', search);
-    
-    const localAuthority = elements.localAuthorityFilter.value;
-    if (localAuthority) params.append('local_authority', localAuthority);
-    
-    const type = elements.typeFilter.value;
-    if (type) params.append('school_type', type);
-    
-    const queryString = params.toString();
-    const endpoint = `/api/schools${queryString ? '?' + queryString : ''}`;
-    
-    const data = await fetchAPI(endpoint);
-    if (!data) {
-        showEmptyState(elements.schoolsGrid, 'Unable to load schools');
-        return;
-    }
-    
-    allSchools = data.schools;
-    renderSchools(allSchools);
-}
-
-async function loadSchoolDetails(urn) {
-    const data = await fetchAPI(`/api/schools/${urn}`);
-    return data;
-}
-
-async function loadComparison() {
-    if (selectedSchools.length === 0) return null;
-    
-    const urns = selectedSchools.map(s => s.urn).join(',');
-    const data = await fetchAPI(`/api/compare?urns=${urns}`);
-    return data;
-}
-
-async function loadRankingYears() {
-    const data = await fetchAPI('/api/filters');
-    if (!data) return;
     
+    // Populate ranking year dropdown
     elements.rankingYear.innerHTML = '';
     data.years.sort((a, b) => b - a).forEach(year => {
         const option = document.createElement('option');
@@ -146,6 +221,116 @@ async function loadRankingYears() {
     });
 }
 
+// =============================================================================
+// DATA LOADING
+// =============================================================================
+
+async function loadSchools() {
+    const search = elements.schoolSearch.value.trim();
+    const localAuthority = elements.localAuthorityFilter.value;
+    const type = elements.typeFilter.value;
+    
+    // If no search query (or less than 2 chars) and no filters, show featured schools
+    if (search.length < 2 && !localAuthority && !type) {
+        await loadFeaturedSchools();
+        return;
+    }
+    
+    const params = new URLSearchParams();
+    
+    if (search.length >= 2) params.append('search', search);
+    if (localAuthority) params.append('local_authority', localAuthority);
+    if (type) params.append('school_type', type);
+    
+    params.append('page', state.pagination.page);
+    params.append('page_size', state.pagination.pageSize);
+    
+    const queryString = params.toString();
+    const endpoint = `/api/schools?${queryString}`;
+    
+    // Don't cache search results (they change based on input)
+    const data = await fetchAPI(endpoint, { useCache: false, showLoading: 'schools' });
+    
+    if (!data) {
+        showEmptyState(elements.schoolsGrid, 'Unable to load schools');
+        return;
+    }
+    
+    state.schools = data.schools;
+    state.pagination.total = data.total;
+    state.pagination.totalPages = data.total_pages;
+    state.isShowingFeatured = false;
+    
+    renderSchools(state.schools);
+}
+
+async function loadFeaturedSchools() {
+    // Load a sample of schools and pick 3 random ones
+    const data = await fetchAPI('/api/schools?page_size=100', { showLoading: 'schools' });
+    
+    if (!data || !data.schools.length) {
+        showEmptyState(elements.schoolsGrid, 'No schools available');
+        return;
+    }
+    
+    // Shuffle and pick 3 random schools
+    const shuffled = data.schools.sort(() => Math.random() - 0.5);
+    state.schools = shuffled.slice(0, 3);
+    state.isShowingFeatured = true;
+    
+    renderFeaturedSchools(state.schools);
+}
+
+function renderFeaturedSchools(schools) {
+    elements.schoolsGrid.innerHTML = `
+        <div class="featured-header">
+            <h3>Featured Schools</h3>
+            <p>Start typing to search ${state.filters?.local_authorities?.length || ''} schools across England</p>
+        </div>
+        ${schools.map(school => `
+            <div class="school-card featured" data-urn="${school.urn}">
+                <h3 class="school-name">${escapeHtml(school.school_name)}</h3>
+                <div class="school-meta">
+                    <span class="school-tag">${escapeHtml(school.local_authority || '')}</span>
+                    <span class="school-tag type">${escapeHtml(school.school_type || '')}</span>
+                </div>
+                <div class="school-address">${escapeHtml(school.address || '')}</div>
+                <div class="school-stats">
+                    <div class="stat">
+                        <div class="stat-value">Primary</div>
+                        <div class="stat-label">Phase</div>
+                    </div>
+                    <div class="stat">
+                        <div class="stat-value">KS2</div>
+                        <div class="stat-label">Data</div>
+                    </div>
+                </div>
+            </div>
+        `).join('')}
+    `;
+    
+    // Add click handlers
+    elements.schoolsGrid.querySelectorAll('.school-card').forEach(card => {
+        card.addEventListener('click', () => {
+            const urn = parseInt(card.dataset.urn);
+            openSchoolModal(urn);
+        });
+    });
+}
+
+async function loadSchoolDetails(urn) {
+    const data = await fetchAPI(`/api/schools/${urn}`, { showLoading: 'modal' });
+    return data;
+}
+
+async function loadComparison() {
+    if (state.selectedSchools.length === 0) return null;
+    
+    const urns = state.selectedSchools.map(s => s.urn).join(',');
+    const data = await fetchAPI(`/api/compare?urns=${urns}`, { useCache: false, showLoading: 'comparison' });
+    return data;
+}
+
 async function loadRankings() {
     const metric = elements.rankingMetric.value;
     const year = elements.rankingYear.value;
@@ -153,7 +338,8 @@ async function loadRankings() {
     let endpoint = `/api/rankings?metric=${metric}&limit=20`;
     if (year) endpoint += `&year=${year}`;
     
-    const data = await fetchAPI(endpoint);
+    const data = await fetchAPI(endpoint, { showLoading: 'rankings' });
+    
     if (!data) {
         showEmptyState(elements.rankingsList, 'Unable to load rankings');
         return;
@@ -162,14 +348,47 @@ async function loadRankings() {
     renderRankings(data.rankings, metric);
 }
 
-// Render Functions
+// =============================================================================
+// METRIC HELPERS
+// =============================================================================
+
+function getMetricLabel(key, short = false) {
+    if (state.metrics) {
+        const metric = state.metrics.find(m => m.key === key);
+        if (metric) {
+            return short ? metric.short_name : metric.name;
+        }
+    }
+    // Fallback labels
+    return key.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
+}
+
+function formatMetricValue(value, metric) {
+    if (value === null || value === undefined) return '-';
+    
+    const metricDef = state.metrics?.find(m => m.key === metric);
+    const type = metricDef?.type || (metric.includes('pct') ? 'percentage' : 'score');
+    
+    if (metric.includes('progress')) {
+        return (value >= 0 ? '+' : '') + value.toFixed(1);
+    }
+    if (type === 'percentage' || metric.includes('pct')) {
+        return value.toFixed(0) + '%';
+    }
+    return value.toFixed(1);
+}
+
+// =============================================================================
+// RENDER FUNCTIONS
+// =============================================================================
+
 function renderSchools(schools) {
     if (schools.length === 0) {
         showEmptyState(elements.schoolsGrid, 'No primary schools found matching your criteria');
         return;
     }
     
-    elements.schoolsGrid.innerHTML = schools.map(school => `
+    let html = schools.map(school => `
         <div class="school-card" data-urn="${school.urn}">
             <h3 class="school-name">${escapeHtml(school.school_name)}</h3>
             <div class="school-meta">
@@ -190,6 +409,19 @@ function renderSchools(schools) {
         </div>
     `).join('');
     
+    // Add pagination info
+    if (state.pagination.totalPages > 1) {
+        html += `
+            <div class="pagination-info">
+                <span>Showing ${schools.length} of ${state.pagination.total} schools</span>
+                ${state.pagination.page < state.pagination.totalPages ? 
+                    `<button class="btn-load-more" onclick="loadMoreSchools()">Load More</button>` : ''}
+            </div>
+        `;
+    }
+    
+    elements.schoolsGrid.innerHTML = html;
+    
     // Add click handlers
     elements.schoolsGrid.querySelectorAll('.school-card').forEach(card => {
         card.addEventListener('click', () => {
@@ -199,69 +431,40 @@ function renderSchools(schools) {
     });
 }
 
+async function loadMoreSchools() {
+    state.pagination.page++;
+    const params = new URLSearchParams();
+    
+    const search = elements.schoolSearch.value.trim();
+    if (search) params.append('search', search);
+    
+    const localAuthority = elements.localAuthorityFilter.value;
+    if (localAuthority) params.append('local_authority', localAuthority);
+    
+    const type = elements.typeFilter.value;
+    if (type) params.append('school_type', type);
+    
+    params.append('page', state.pagination.page);
+    params.append('page_size', state.pagination.pageSize);
+    
+    const data = await fetchAPI(`/api/schools?${params.toString()}`, { useCache: false });
+    
+    if (data && data.schools.length > 0) {
+        state.schools = [...state.schools, ...data.schools];
+        renderSchools(state.schools);
+    }
+}
+
 function renderRankings(rankings, metric) {
     if (rankings.length === 0) {
         showEmptyState(elements.rankingsList, 'No ranking data available for this year/metric');
         return;
     }
     
-    const metricLabels = {
-        // Expected
-        rwm_expected_pct: 'RWM %',
-        reading_expected_pct: 'Reading %',
-        writing_expected_pct: 'Writing %',
-        maths_expected_pct: 'Maths %',
-        gps_expected_pct: 'GPS %',
-        science_expected_pct: 'Science %',
-        // Higher
-        rwm_high_pct: 'RWM Higher %',
-        reading_high_pct: 'Reading Higher %',
-        writing_high_pct: 'Writing Higher %',
-        maths_high_pct: 'Maths Higher %',
-        gps_high_pct: 'GPS Higher %',
-        // Progress
-        reading_progress: 'Reading Progress',
-        writing_progress: 'Writing Progress',
-        maths_progress: 'Maths Progress',
-        // Averages
-        reading_avg_score: 'Reading Avg',
-        maths_avg_score: 'Maths Avg',
-        gps_avg_score: 'GPS Avg',
-        // Gender
-        rwm_expected_boys_pct: 'Boys RWM %',
-        rwm_expected_girls_pct: 'Girls RWM %',
-        rwm_high_boys_pct: 'Boys Higher %',
-        rwm_high_girls_pct: 'Girls Higher %',
-        // Equity
-        rwm_expected_disadvantaged_pct: 'Disadvantaged %',
-        rwm_expected_non_disadvantaged_pct: 'Non-Disadv %',
-        disadvantaged_gap: 'Disadv Gap',
-        // Context
-        disadvantaged_pct: '% Disadvantaged',
-        eal_pct: '% EAL',
-        sen_support_pct: '% SEN',
-        stability_pct: '% Stable',
-        // 3-Year
-        rwm_expected_3yr_pct: 'RWM 3yr %',
-        reading_avg_3yr: 'Reading 3yr',
-        maths_avg_3yr: 'Maths 3yr',
-    };
-    
     elements.rankingsList.innerHTML = rankings.map((school, index) => {
         const value = school[metric];
         if (value === null || value === undefined) return '';
         
-        const isProgress = metric.includes('progress');
-        const isScore = metric.includes('_avg_');
-        let displayValue;
-        if (isProgress) {
-            displayValue = (value >= 0 ? '+' : '') + value.toFixed(1);
-        } else if (isScore) {
-            displayValue = value.toFixed(0);
-        } else {
-            displayValue = value.toFixed(0) + '%';
-        }
-        
         return `
             <div class="ranking-item" data-urn="${school.urn}">
                 <div class="ranking-position ${index < 3 ? 'top-3' : ''}">${index + 1}</div>
@@ -270,8 +473,8 @@ function renderRankings(rankings, metric) {
                     <div class="ranking-location">${escapeHtml(school.local_authority || '')}</div>
                 </div>
                 <div class="ranking-score">
-                    <div class="ranking-score-value">${displayValue}</div>
+                    <div class="ranking-score-value">${formatMetricValue(value, metric)}</div>
-                    <div class="ranking-score-label">${metricLabels[metric] || metric}</div>
+                    <div class="ranking-score-label">${getMetricLabel(metric, true)}</div>
                 </div>
             </div>
         `;
@@ -287,7 +490,7 @@ function renderRankings(rankings, metric) {
 }
 
 function renderSelectedSchools() {
-    if (selectedSchools.length === 0) {
+    if (state.selectedSchools.length === 0) {
         elements.selectedSchools.innerHTML = `
             <div class="empty-selection">
                 <div class="empty-icon">
@@ -305,7 +508,7 @@ function renderSelectedSchools() {
         return;
     }
     
-    elements.selectedSchools.innerHTML = selectedSchools.map((school, index) => `
+    elements.selectedSchools.innerHTML = state.selectedSchools.map((school, index) => `
         <div class="selected-school-tag" style="border-left: 3px solid ${CHART_COLORS[index % CHART_COLORS.length]}">
             <span>${escapeHtml(school.school_name)}</span>
             <button class="remove" data-urn="${school.urn}" title="Remove">
@@ -330,59 +533,18 @@ function renderSelectedSchools() {
 }
 
 async function updateComparisonChart() {
-    if (selectedSchools.length === 0) return;
+    if (state.selectedSchools.length === 0) return;
     
     const data = await loadComparison();
     if (!data) return;
     
     const metric = elements.metricSelect.value;
-    const metricLabels = {
-        // Expected Standard
-        rwm_expected_pct: 'Reading, Writing & Maths Combined (%)',
-        reading_expected_pct: 'Reading Expected Standard (%)',
-        writing_expected_pct: 'Writing Expected Standard (%)',
-        maths_expected_pct: 'Maths Expected Standard (%)',
-        gps_expected_pct: 'GPS Expected Standard (%)',
-        science_expected_pct: 'Science Expected Standard (%)',
-        // Higher Standard
-        rwm_high_pct: 'RWM Combined Higher Standard (%)',
-        reading_high_pct: 'Reading Higher Standard (%)',
-        writing_high_pct: 'Writing Greater Depth (%)',
-        maths_high_pct: 'Maths Higher Standard (%)',
-        gps_high_pct: 'GPS Higher Standard (%)',
-        // Progress
-        reading_progress: 'Reading Progress Score',
-        writing_progress: 'Writing Progress Score',
-        maths_progress: 'Maths Progress Score',
-        // Averages
-        reading_avg_score: 'Reading Average Scaled Score',
-        maths_avg_score: 'Maths Average Scaled Score',
-        gps_avg_score: 'GPS Average Scaled Score',
-        // Gender
-        rwm_expected_boys_pct: 'RWM Expected % (Boys)',
-        rwm_expected_girls_pct: 'RWM Expected % (Girls)',
-        rwm_high_boys_pct: 'RWM Higher % (Boys)',
-        rwm_high_girls_pct: 'RWM Higher % (Girls)',
-        // Equity
-        rwm_expected_disadvantaged_pct: 'RWM Expected % (Disadvantaged)',
-        rwm_expected_non_disadvantaged_pct: 'RWM Expected % (Non-Disadvantaged)',
-        disadvantaged_gap: 'Disadvantaged Gap vs National',
-        // Context
-        disadvantaged_pct: '% Disadvantaged Pupils',
-        eal_pct: '% EAL Pupils',
-        sen_support_pct: '% SEN Support',
-        stability_pct: '% Pupil Stability',
-        // 3-Year
-        rwm_expected_3yr_pct: 'RWM Expected % (3-Year Avg)',
-        reading_avg_3yr: 'Reading Score (3-Year Avg)',
-        maths_avg_3yr: 'Maths Score (3-Year Avg)',
-    };
     
-    // Prepare chart data - iterate in same order as selectedSchools for color consistency
+    // Prepare chart data
     const datasets = [];
     const allYears = new Set();
     
-    selectedSchools.forEach((school, index) => {
+    state.selectedSchools.forEach((school, index) => {
         const schoolData = data.comparison[school.urn];
         if (!schoolData) return;
         
@@ -434,7 +596,7 @@ async function updateComparisonChart() {
                 },
                 title: {
                     display: true,
-                    text: metricLabels[metric],
+                    text: getMetricLabel(metric),
                     font: { family: "'Playfair Display', serif", size: 18, weight: 600 },
                     padding: { bottom: 20 },
                 },
@@ -458,7 +620,7 @@ async function updateComparisonChart() {
                 y: {
                     title: {
                         display: true,
-                        text: metricLabels[metric],
+                        text: getMetricLabel(metric),
                         font: { family: "'DM Sans', sans-serif", weight: 500 },
                     },
                     grid: { color: '#e5dfd5' },
@@ -492,9 +654,9 @@ function updateComparisonTable(comparison, metric, years) {
     }
     elements.tableHeader.innerHTML = headerHtml;
     
-    // Build body - iterate in same order as selectedSchools for color consistency
+    // Build body
     let bodyHtml = '';
-    selectedSchools.forEach((school, index) => {
+    state.selectedSchools.forEach((school, index) => {
         const schoolData = comparison[school.urn];
         if (!schoolData) return;
         
@@ -511,7 +673,7 @@ function updateComparisonTable(comparison, metric, years) {
         const oneYearChangeStr = oneYearChange !== null ? oneYearChange.toFixed(1) : 'N/A';
         const oneYearClass = oneYearChange !== null ? (oneYearChange >= 0 ? 'positive' : 'negative') : '';
         
-        // Calculate variability (standard deviation) - exclude null/0 values
+        // Calculate variability (standard deviation)
         const values = years.map(y => yearlyMap[y]).filter(v => v != null && v !== 0);
         let variabilityStr = 'N/A';
         if (values.length >= 2) {
@@ -541,22 +703,21 @@ function updateComparisonTable(comparison, metric, years) {
     elements.tableBody.innerHTML = bodyHtml;
 }
 
-function formatMetricValue(value, metric) {
-    if (value === null || value === undefined) return '-';
-    if (metric.includes('progress')) {
-        return (value >= 0 ? '+' : '') + value.toFixed(1);
-    }
-    if (metric.includes('pct')) {
-        return value.toFixed(0) + '%';
-    }
-    return value.toFixed(1);
-}
-
 async function openSchoolModal(urn) {
-    const data = await loadSchoolDetails(urn);
+    // Show loading state immediately
-    if (!data) return;
+    elements.modal.classList.add('active');
+    document.body.style.overflow = 'hidden';
+    elements.modalStats.innerHTML = '<div class="loading"><div class="loading-spinner"></div><p>Loading school data...</p></div>';
+    elements.modalSchoolName.textContent = 'Loading...';
+    elements.modalMeta.innerHTML = '';
     
-    currentSchoolData = data;
+    const data = await loadSchoolDetails(urn);
+    if (!data) {
+        elements.modalStats.innerHTML = '<div class="empty-state"><p>Unable to load school data</p></div>';
+        return;
+    }
+    
+    state.currentSchoolData = data;
     
     elements.modalSchoolName.textContent = data.school_info.school_name;
     elements.modalMeta.innerHTML = `
@@ -565,7 +726,7 @@ async function openSchoolModal(urn) {
         <span class="school-tag">Primary</span>
     `;
     
-    // Get latest year data with actual results (skip 2021 - no SATs)
+    // Get latest year data with actual results
     const sortedData = data.yearly_data.sort((a, b) => b.year - a.year);
     const latest = sortedData.find(d => d.rwm_expected_pct !== null) || sortedData[0];
     
@@ -636,7 +797,7 @@ async function openSchoolModal(urn) {
         return value >= 0 ? 'positive' : 'negative';
     }
     
-    // Create chart - filter out years with no data (2021)
+    // Create chart
     if (schoolDetailChart) {
         schoolDetailChart.destroy();
     }
@@ -702,33 +863,30 @@ async function openSchoolModal(urn) {
     });
     
     // Update add to compare button
-    const isSelected = selectedSchools.some(s => s.urn === data.school_info.urn);
+    const isSelected = state.selectedSchools.some(s => s.urn === data.school_info.urn);
     elements.addToCompare.textContent = isSelected ? 'Remove from Compare' : 'Add to Compare';
     elements.addToCompare.dataset.urn = data.school_info.urn;
-    
-    elements.modal.classList.add('active');
-    document.body.style.overflow = 'hidden';
 }
 
 function closeModal() {
     elements.modal.classList.remove('active');
     document.body.style.overflow = '';
-    currentSchoolData = null;
+    state.currentSchoolData = null;
 }
 
 function addToComparison(school) {
-    if (selectedSchools.some(s => s.urn === school.urn)) return;
+    if (state.selectedSchools.some(s => s.urn === school.urn)) return;
-    if (selectedSchools.length >= 5) {
+    if (state.selectedSchools.length >= 5) {
         alert('Maximum 5 schools can be compared at once');
         return;
     }
     
-    selectedSchools.push(school);
+    state.selectedSchools.push(school);
     renderSelectedSchools();
 }
 
 function removeFromComparison(urn) {
-    selectedSchools = selectedSchools.filter(s => s.urn !== urn);
+    state.selectedSchools = state.selectedSchools.filter(s => s.urn !== urn);
     renderSelectedSchools();
 }
 
@@ -751,7 +909,10 @@ function escapeHtml(text) {
     return div.innerHTML;
 }
 
-// Event Listeners
+// =============================================================================
+// EVENT LISTENERS
+// =============================================================================
+
 function setupEventListeners() {
     // Navigation
     document.querySelectorAll('.nav-link').forEach(link => {
@@ -771,11 +932,18 @@ function setupEventListeners() {
     let searchTimeout;
     elements.schoolSearch.addEventListener('input', () => {
         clearTimeout(searchTimeout);
+        state.pagination.page = 1; // Reset to first page on new search
         searchTimeout = setTimeout(loadSchools, 300);
     });
     
-    elements.localAuthorityFilter.addEventListener('change', loadSchools);
+    elements.localAuthorityFilter.addEventListener('change', () => {
-    elements.typeFilter.addEventListener('change', loadSchools);
+        state.pagination.page = 1;
+        loadSchools();
+    });
+    elements.typeFilter.addEventListener('change', () => {
+        state.pagination.page = 1;
+        loadSchools();
+    });
     
     // Compare search
     let compareSearchTimeout;
@@ -785,7 +953,7 @@ function setupEventListeners() {
         if (!data) return;
         lastCompareSearchData = data;
         
-        const results = data.schools.filter(s => !selectedSchools.some(sel => sel.urn === s.urn));
+        const results = data.schools.filter(s => !state.selectedSchools.some(sel => sel.urn === s.urn));
         
         const headerHtml = `
             <div class="compare-results-header">
@@ -814,7 +982,6 @@ function setupEventListeners() {
                     const school = data.schools.find(s => s.urn === urn);
                     if (school) {
                         addToComparison(school);
-                        // Re-render results without closing (filter out newly added school)
                         renderCompareResults(data);
                     }
                 });
@@ -841,7 +1008,7 @@ function setupEventListeners() {
         }
         
         compareSearchTimeout = setTimeout(async () => {
-            const data = await fetchAPI(`/api/schools?search=${encodeURIComponent(query)}`);
+            const data = await fetchAPI(`/api/schools?search=${encodeURIComponent(query)}`, { useCache: false });
             if (!data) return;
             
             renderCompareResults(data);
@@ -868,16 +1035,16 @@ function setupEventListeners() {
     elements.modal.querySelector('.modal-backdrop').addEventListener('click', closeModal);
     
     elements.addToCompare.addEventListener('click', () => {
-        if (!currentSchoolData) return;
+        if (!state.currentSchoolData) return;
         
-        const urn = currentSchoolData.school_info.urn;
+        const urn = state.currentSchoolData.school_info.urn;
-        const isSelected = selectedSchools.some(s => s.urn === urn);
+        const isSelected = state.selectedSchools.some(s => s.urn === urn);
         
         if (isSelected) {
             removeFromComparison(urn);
             elements.addToCompare.textContent = 'Add to Compare';
         } else {
-            addToComparison(currentSchoolData.school_info);
+            addToComparison(state.currentSchoolData.school_info);
             elements.addToCompare.textContent = 'Remove from Compare';
         }
     });
@@ -885,15 +1052,12 @@ function setupEventListeners() {
     // Keyboard
     document.addEventListener('keydown', (e) => {
         if (e.key === 'Escape') {
-            // Close compare results if open
             if (elements.compareResults.classList.contains('active')) {
                 elements.compareResults.classList.remove('active');
                 elements.compareSearch.value = '';
                 return;
             }
-            // Close modal if open
             closeModal();
         }
     });
 }
-

frontend/index.html

@@ -41,7 +41,7 @@
         <section id="dashboard-view" class="view active">
             <div class="hero">
                 <h1 class="hero-title">Compare Primary School Performance</h1>
-                <p class="hero-subtitle">Explore and compare KS2 results across England's primary schools</p>
+                <p class="hero-subtitle">Search and compare KS2 results across England's primary schools</p>
             </div>
 
             <div class="search-section">

frontend/styles.css

@@ -269,6 +269,31 @@ body {
     gap: 1.5rem;
 }
 
+/* Featured Schools Header */
+.featured-header {
+    grid-column: 1 / -1;
+    text-align: center;
+    padding: 1rem 0 2rem;
+}
+
+.featured-header h3 {
+    font-family: 'Playfair Display', Georgia, serif;
+    font-size: 1.5rem;
+    font-weight: 600;
+    color: var(--text-primary);
+    margin-bottom: 0.5rem;
+}
+
+.featured-header p {
+    color: var(--text-muted);
+    font-size: 0.95rem;
+}
+
+.school-card.featured {
+    border-color: var(--accent-coral);
+    border-width: 2px;
+}
+
 .school-card {
     background: var(--bg-card);
     border: 1px solid var(--border-color);
@@ -922,6 +947,91 @@ body {
     opacity: 0.5;
 }
 
+/* Skeleton Loading */
+.skeleton {
+    pointer-events: none;
+}
+
+.school-card.skeleton {
+    background: var(--bg-card);
+    padding: 1.5rem;
+}
+
+.skeleton-line {
+    height: 1rem;
+    background: linear-gradient(90deg, var(--bg-secondary) 25%, var(--border-color) 50%, var(--bg-secondary) 75%);
+    background-size: 200% 100%;
+    animation: shimmer 1.5s infinite;
+    border-radius: var(--radius-sm);
+    margin-bottom: 0.75rem;
+}
+
+.skeleton-line.short {
+    height: 0.75rem;
+}
+
+.ranking-item.skeleton {
+    background: var(--bg-card);
+}
+
+.skeleton-circle {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+    background: linear-gradient(90deg, var(--bg-secondary) 25%, var(--border-color) 50%, var(--bg-secondary) 75%);
+    background-size: 200% 100%;
+    animation: shimmer 1.5s infinite;
+}
+
+.skeleton-content {
+    flex: 1;
+    padding: 0 1rem;
+}
+
+.skeleton-score {
+    width: 60px;
+    height: 40px;
+    background: linear-gradient(90deg, var(--bg-secondary) 25%, var(--border-color) 50%, var(--bg-secondary) 75%);
+    background-size: 200% 100%;
+    animation: shimmer 1.5s infinite;
+    border-radius: var(--radius-sm);
+}
+
+@keyframes shimmer {
+    0% { background-position: 200% 0; }
+    100% { background-position: -200% 0; }
+}
+
+/* Pagination */
+.pagination-info {
+    grid-column: 1 / -1;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    gap: 1rem;
+    padding: 2rem;
+    color: var(--text-muted);
+    font-size: 0.9rem;
+}
+
+.btn-load-more {
+    padding: 0.6rem 1.5rem;
+    font-size: 0.9rem;
+    font-family: inherit;
+    font-weight: 500;
+    border: 2px solid var(--accent-coral);
+    border-radius: var(--radius-md);
+    background: transparent;
+    color: var(--accent-coral);
+    cursor: pointer;
+    transition: var(--transition);
+}
+
+.btn-load-more:hover {
+    background: var(--accent-coral);
+    color: white;
+}
+
 /* Responsive */
 @media (max-width: 768px) {
     .header-content {

requirements.txt

@@ -3,4 +3,5 @@ uvicorn[standard]==0.27.0
 pandas==2.1.4
 python-multipart==0.0.6
 aiofiles==23.2.1
+pydantic-settings==2.1.0