Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 TudorandClaude Opus 5
|
1d8858fbda | ||
|
TudorandClaude Opus 5
|
eaf5e5d180 | ||
 tudor
|
be780ebe13 | ||
|
TudorandClaude Opus 5
|
b6c2cd5116 | ||
|
tudor
|
dc79d653e5 | ||
|
TudorandClaude Opus 5
|
b0d5334e06 | ||
|
TudorandClaude Opus 5
|
d65eb58883 | ||
|
TudorandClaude Opus 5
|
7f5f0fb676 | ||
|
tudor
|
47f3591ed8 | ||
|
TudorandClaude Opus 5
|
6fc7fce948 | ||
|
tudor
|
eb6d918650 | ||
|
TudorandClaude Opus 5
|
d47ac71c47 | ||
|
tudor
|
17e5371e9c |
No files matched your search
+13
-5
@@ -20,7 +20,7 @@ PORT=80
|
||||
# =============================================================================
|
||||
# CORS
|
||||
# =============================================================================
|
||||
# Comma-separated list of allowed origins
|
||||
# JSON array of allowed origins (pydantic-settings format)
|
||||
# In production, only include your actual domain
|
||||
ALLOWED_ORIGINS=["https://schoolcompare.co.uk"]
|
||||
|
||||
@@ -33,13 +33,21 @@ ADMIN_API_KEY=CHANGE_THIS_TO_A_SECURE_RANDOM_KEY
|
||||
|
||||
# Rate limiting (requests per minute per IP)
|
||||
RATE_LIMIT_PER_MINUTE=60
|
||||
RATE_LIMIT_BURST=10
|
||||
GLOBAL_RATE_LIMIT_PER_MINUTE=3000
|
||||
|
||||
# Maximum request body size in bytes (default 1MB)
|
||||
MAX_REQUEST_SIZE=1048576
|
||||
|
||||
# =============================================================================
|
||||
# API
|
||||
# SEARCH AND OPTIONAL FEATURE FLAGS
|
||||
# =============================================================================
|
||||
DEFAULT_PAGE_SIZE=50
|
||||
MAX_PAGE_SIZE=100
|
||||
TYPESENSE_URL=http://localhost:8108
|
||||
TYPESENSE_API_KEY=CHANGE_THIS_TO_YOUR_TYPESENSE_KEY
|
||||
|
||||
# Empty URL disables Unleash-backed flags. Match the managed environment when used.
|
||||
UNLEASH_URL=
|
||||
UNLEASH_API_TOKEN=
|
||||
|
||||
# Page-size limits are currently declared by route Query parameters.
|
||||
# DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE and RATE_LIMIT_BURST are not reliable tuning
|
||||
# controls in the current routes; see docs/LEGACY_CODE.md.
|
||||
+13
-187
@@ -1,191 +1,17 @@
|
||||
# Docker Deployment Guide
|
||||
# Docker deployment
|
||||
|
||||
## Quick Start
|
||||
The maintained deployment runbook is [docs/DEPLOY.md](docs/DEPLOY.md).
|
||||
|
||||
Deploy the complete SchoolCompare stack (PostgreSQL + FastAPI + Next.js) with one command:
|
||||
- Production: `docker-compose.portainer.yml`, using `:prod` images.
|
||||
- Staging: `docker-compose.portainer.staging.yml`, using `:staging` images.
|
||||
- Builds and deployment: `.gitea/workflows/deploy.yml`.
|
||||
- Human-approved production promotion: `.gitea/workflows/promote.yml`.
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
The generic `docker-compose.yml` is not a supported one-command onboarding path:
|
||||
it still uses `:latest` tags that the release workflow no longer publishes and
|
||||
lacks the full current CMS setup. Review the [legacy inventory](docs/LEGACY_CODE.md)
|
||||
before using old compose examples. Starting an empty database does not populate
|
||||
school marts.
|
||||
|
||||
This will start:
|
||||
- **PostgreSQL** on port 5432 (database)
|
||||
- **FastAPI** on port 8000 (backend API)
|
||||
- **Next.js** on port 3000 (frontend)
|
||||
|
||||
## Service Details
|
||||
|
||||
### PostgreSQL Database
|
||||
- **Port**: 5432
|
||||
- **Container**: `schoolcompare_db`
|
||||
- **Credentials**:
|
||||
- User: `schoolcompare`
|
||||
- Password: `schoolcompare`
|
||||
- Database: `schoolcompare`
|
||||
- **Volume**: `postgres_data` (persistent storage)
|
||||
|
||||
### FastAPI Backend
|
||||
- **Port**: 8000 → 80 (container)
|
||||
- **Container**: `schoolcompare_backend`
|
||||
- **Built from**: Root `Dockerfile`
|
||||
- **API Endpoint**: http://localhost:8000/api
|
||||
- **Health Check**: http://localhost:8000/api/data-info
|
||||
|
||||
### Next.js Frontend
|
||||
- **Port**: 3000
|
||||
- **Container**: `schoolcompare_nextjs`
|
||||
- **Built from**: `nextjs-app/Dockerfile`
|
||||
- **URL**: http://localhost:3000
|
||||
- **Connects to**: Backend via internal network
|
||||
|
||||
## Commands
|
||||
|
||||
### Start all services
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### View logs
|
||||
```bash
|
||||
# All services
|
||||
docker-compose logs -f
|
||||
|
||||
# Specific service
|
||||
docker-compose logs -f nextjs
|
||||
docker-compose logs -f backend
|
||||
docker-compose logs -f db
|
||||
```
|
||||
|
||||
### Check status
|
||||
```bash
|
||||
docker-compose ps
|
||||
```
|
||||
|
||||
### Stop all services
|
||||
```bash
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
### Rebuild after code changes
|
||||
```bash
|
||||
# Rebuild and restart specific service
|
||||
docker-compose up -d --build nextjs
|
||||
|
||||
# Rebuild all services
|
||||
docker-compose up -d --build
|
||||
```
|
||||
|
||||
### Clean restart (remove volumes)
|
||||
```bash
|
||||
docker-compose down -v
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
## Initial Database Setup
|
||||
|
||||
After first start, you may need to initialize the database:
|
||||
|
||||
```bash
|
||||
# Enter the backend container
|
||||
docker exec -it schoolcompare_backend bash
|
||||
|
||||
# Run migrations or data loading
|
||||
python -m backend.data_loader
|
||||
```
|
||||
|
||||
## Accessing Services
|
||||
|
||||
Once running:
|
||||
- **Frontend**: http://localhost:3000
|
||||
- **Backend API**: http://localhost:8000/api
|
||||
- **API Docs**: http://localhost:8000/docs (Swagger UI)
|
||||
- **Database**: localhost:5432 (use any PostgreSQL client)
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Create a `.env` file in the root directory to customize:
|
||||
|
||||
```env
|
||||
# Database
|
||||
POSTGRES_USER=schoolcompare
|
||||
POSTGRES_PASSWORD=your_secure_password
|
||||
POSTGRES_DB=schoolcompare
|
||||
|
||||
# Backend
|
||||
DATABASE_URL=postgresql://schoolcompare:your_secure_password@db:5432/schoolcompare
|
||||
|
||||
# Frontend (for client-side access)
|
||||
NEXT_PUBLIC_API_URL=http://localhost:8000/api
|
||||
```
|
||||
|
||||
Then run:
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Backend not connecting to database
|
||||
```bash
|
||||
# Check database health
|
||||
docker-compose ps
|
||||
|
||||
# View backend logs
|
||||
docker-compose logs backend
|
||||
|
||||
# Restart backend
|
||||
docker-compose restart backend
|
||||
```
|
||||
|
||||
### Frontend not connecting to backend
|
||||
```bash
|
||||
# Check backend health
|
||||
curl http://localhost:8000/api/data-info
|
||||
|
||||
# Check Next.js environment variables
|
||||
docker exec schoolcompare_nextjs env | grep API
|
||||
```
|
||||
|
||||
### Port already in use
|
||||
```bash
|
||||
# Change ports in docker-compose.yml
|
||||
# For example, change "3000:3000" to "3001:3000"
|
||||
```
|
||||
|
||||
### Rebuild from scratch
|
||||
```bash
|
||||
docker-compose down -v
|
||||
docker system prune -a
|
||||
docker-compose up -d --build
|
||||
```
|
||||
|
||||
## Production Deployment
|
||||
|
||||
For production, update the following:
|
||||
|
||||
1. **Use secure passwords** in `.env` file
|
||||
2. **Configure reverse proxy** (Nginx) in front of Next.js
|
||||
3. **Enable HTTPS** with SSL certificates
|
||||
4. **Set production environment variables**:
|
||||
```env
|
||||
NODE_ENV=production
|
||||
POSTGRES_PASSWORD=<strong-password>
|
||||
```
|
||||
5. **Backup database** regularly:
|
||||
```bash
|
||||
docker exec schoolcompare_db pg_dump -U schoolcompare schoolcompare > backup.sql
|
||||
```
|
||||
|
||||
## Network Architecture
|
||||
|
||||
```
|
||||
Internet
|
||||
↓
|
||||
Next.js (port 3000) ← User browsers
|
||||
↓ (internal network)
|
||||
FastAPI (port 8000) ← API calls
|
||||
↓ (internal network)
|
||||
PostgreSQL (port 5432) ← Data queries
|
||||
```
|
||||
|
||||
All services communicate via the `schoolcompare-network` Docker network.
|
||||
For architecture, configuration and test commands, see
|
||||
[ARCHITECTURE.md](docs/ARCHITECTURE.md) and [DEVELOPMENT.md](docs/DEVELOPMENT.md).
|
||||
@@ -1,3 +1,5 @@
|
||||
> Historical migration record, retained for context. Setup and architecture claims below may be obsolete. Use [README.md](README.md), [architecture](docs/ARCHITECTURE.md) and [deployment](docs/DEPLOY.md) for current guidance.
|
||||
|
||||
# SchoolCompare: Vanilla JS → Next.js Migration Summary
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -1,214 +1,67 @@
|
||||
# Primary School Compass 🧒📚
|
||||
# SchoolCompare
|
||||
|
||||
A modern web application for comparing **primary school (KS2)** performance data in **Wandsworth and Merton** over the last 5 years. Built with FastAPI and vanilla JavaScript with Chart.js visualizations.
|
||||
SchoolCompare compares schools across England: primary (KS2), secondary (KS4),
|
||||
all-through and post-16 provision, with coverage depending on the source dataset.
|
||||
It provides school search, postcode maps, comparisons, rankings, place pages,
|
||||
Ofsted information, admissions and destination measures. Editorial content lives
|
||||
in a Payload CMS blog.
|
||||
|
||||
|
||||
|
||||
|
||||
## Start here
|
||||
|
||||
## Features
|
||||
- [Architecture and data flow](docs/ARCHITECTURE.md)
|
||||
- [Development and validation](docs/DEVELOPMENT.md)
|
||||
- [Deployment and promotion](docs/DEPLOY.md)
|
||||
- [Legacy and unused-code inventory](docs/LEGACY_CODE.md)
|
||||
- [Frontend conventions](nextjs-app/README.md)
|
||||
- [CMS publishing](nextjs-app/docs/PUBLISHING.md)
|
||||
|
||||
- 📊 **Interactive Charts** - Visualize KS2 performance trends over time
|
||||
- 🔍 **Smart Search** - Find primary schools by name in Wandsworth & Merton
|
||||
- ⚖️ **Side-by-Side Comparison** - Compare up to 5 schools simultaneously
|
||||
- 🏆 **Rankings** - View top-performing primary schools by various KS2 metrics
|
||||
- 📱 **Responsive Design** - Works beautifully on desktop and mobile
|
||||
## Repository map
|
||||
|
||||
## Key Metrics (KS2)
|
||||
| Path | Responsibility |
|
||||
|---|---|
|
||||
| `backend/` | FastAPI routes, cached school data, read-only SQLAlchemy mappings, feature flags |
|
||||
| `nextjs-app/` | Next.js App Router, React UI, Payload CMS, frontend tests |
|
||||
| `pipeline/plugins/extractors/` | Custom Singer taps for GIAS, EES, Ofsted and other datasets |
|
||||
| `pipeline/transform/` | dbt staging/intermediate models, marts, seeds and data tests |
|
||||
| `pipeline/dags/` | Airflow extraction, transformation and publication workflows |
|
||||
| `pipeline/scripts/` | Search indexing, code generation and operational diagnostics |
|
||||
| `e2e/` | Playwright journeys against a running environment |
|
||||
| `.gitea/workflows/` | PR checks, staging deployment and manual production promotion |
|
||||
| `scripts/` | CI review tooling and historical data utilities; see the legacy inventory |
|
||||
| `docs/superpowers/`, `mockups/` | Design history and prototypes, not application entry points |
|
||||
|
||||
The application tracks these Key Stage 2 performance indicators:
|
||||
## Runtime
|
||||
|
||||
| Metric | Description |
|
||||
|--------|-------------|
|
||||
| **Reading Progress** | Progress in reading from KS1 to KS2 |
|
||||
| **Writing Progress** | Progress in writing from KS1 to KS2 |
|
||||
| **Maths Progress** | Progress in maths from KS1 to KS2 |
|
||||
| **Reading Expected %** | Percentage meeting expected standard in reading |
|
||||
| **Writing Expected %** | Percentage meeting expected standard in writing |
|
||||
| **Maths Expected %** | Percentage meeting expected standard in maths |
|
||||
| **Reading, Writing & Maths Combined %** | Percentage meeting expected standard in all three subjects |
|
||||
The public site is **Next.js**, not the FastAPI root page. Browser `/api/*`
|
||||
requests pass through a Next.js route handler to FastAPI. Server-rendered pages
|
||||
call FastAPI directly using `FASTAPI_URL`, including its `/api` suffix.
|
||||
|
||||
## Quick Start
|
||||
PostgreSQL/PostGIS stores school data. Meltano/Singer extracts source data;
|
||||
dbt builds `marts.*`; FastAPI reads those tables. Typesense serves text search
|
||||
and autocomplete. Payload runs inside Next.js and owns a separate `payload`
|
||||
database schema and uploaded media.
|
||||
|
||||
### 1. Clone and Setup
|
||||
There is **no automatic CSV import or sample dataset on startup**. A working
|
||||
school-data environment needs populated marts from the pipeline or an approved
|
||||
database snapshot. See [development](docs/DEVELOPMENT.md) before choosing a setup.
|
||||
|
||||
```bash
|
||||
cd school_results
|
||||
## Validation
|
||||
|
||||
# Create virtual environment
|
||||
python -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
```sh
|
||||
cd nextjs-app
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test -- --runInBand
|
||||
```
|
||||
|
||||
### 2. Run the Application
|
||||
Backend checks, pipeline validation, runtime versions and E2E requirements are
|
||||
listed in [DEVELOPMENT.md](docs/DEVELOPMENT.md). No `npm run lint` script is
|
||||
currently defined.
|
||||
|
||||
```bash
|
||||
# Start the server
|
||||
python -m uvicorn backend.app:app --reload --port 8000
|
||||
```
|
||||
|
||||
Then open http://localhost:8000 in your browser.
|
||||
|
||||
The app will run with **sample data** by default, showing **110 primary schools** (66 in Wandsworth, 44 in Merton) with 5 years of KS2 performance data.
|
||||
|
||||
### 3. (Optional) Use Real Data
|
||||
|
||||
To use real UK school performance data:
|
||||
|
||||
1. Visit [Compare School Performance - Download Data](https://www.compare-school-performance.service.gov.uk/download-data)
|
||||
|
||||
2. Download **Key Stage 2** data for the years you want (2019-2024)
|
||||
- Select "Key Stage 2" as the data type
|
||||
|
||||
3. Place the CSV files in the `data/` folder
|
||||
|
||||
4. Restart the server - it will automatically load and filter to Wandsworth & Merton schools
|
||||
|
||||
**Note:** The app only displays schools in Wandsworth and Merton. Data from other areas will be filtered out.
|
||||
|
||||
See the helper script for more details:
|
||||
```bash
|
||||
python scripts/download_data.py
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
school_results/
|
||||
├── backend/
|
||||
│ └── app.py # FastAPI application with all API endpoints
|
||||
├── frontend/
|
||||
│ ├── index.html # Main HTML page
|
||||
│ ├── styles.css # Styling (warm, editorial design)
|
||||
│ └── app.js # Frontend JavaScript
|
||||
├── data/
|
||||
│ └── .gitkeep # Place CSV data files here
|
||||
├── scripts/
|
||||
│ └── download_data.py # Helper for downloading/processing data
|
||||
├── requirements.txt # Python dependencies
|
||||
└── README.md
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Endpoint | Description |
|
||||
|----------|-------------|
|
||||
| `GET /api/schools` | List schools with optional search/filter |
|
||||
| `GET /api/schools/{urn}` | Get detailed data for a specific school |
|
||||
| `GET /api/compare?urns=...` | Compare multiple schools |
|
||||
| `GET /api/rankings` | Get school rankings by metric |
|
||||
| `GET /api/filters` | Get available filter options |
|
||||
| `GET /api/metrics` | Get available performance metrics |
|
||||
|
||||
### Example API Usage
|
||||
|
||||
```bash
|
||||
# Search for schools
|
||||
curl "http://localhost:8000/api/schools?search=academy"
|
||||
|
||||
# Get school details
|
||||
curl "http://localhost:8000/api/schools/100001"
|
||||
|
||||
# Compare schools
|
||||
curl "http://localhost:8000/api/compare?urns=100001,100002,100003"
|
||||
|
||||
# Get rankings
|
||||
curl "http://localhost:8000/api/rankings?metric=rwm_expected_pct&year=2024"
|
||||
```
|
||||
|
||||
## Data Format
|
||||
|
||||
If using your own CSV data, ensure it includes these columns (or similar):
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| URN | Integer | Unique Reference Number |
|
||||
| SCHNAME | String | School name |
|
||||
| LA | String | Local Authority (must be Wandsworth or Merton) |
|
||||
| READPROG | Float | Reading progress score |
|
||||
| WRITPROG | Float | Writing progress score |
|
||||
| MATPROG | Float | Maths progress score |
|
||||
| PTRWM_EXP | Float | % meeting expected standard in reading, writing & maths |
|
||||
| PTREAD_EXP | Float | % meeting expected standard in reading |
|
||||
| PTWRIT_EXP | Float | % meeting expected standard in writing |
|
||||
| PTMAT_EXP | Float | % meeting expected standard in maths |
|
||||
|
||||
The application normalizes column names automatically and filters to only show Wandsworth and Merton schools.
|
||||
|
||||
## Technology Stack
|
||||
|
||||
- **Backend**: FastAPI (Python) - High-performance async API framework
|
||||
- **Frontend**: Vanilla JavaScript with Chart.js
|
||||
- **Styling**: Custom CSS with CSS variables for theming
|
||||
- **Data**: Pandas for CSV processing
|
||||
|
||||
## Design Philosophy
|
||||
|
||||
The UI features a warm, editorial design inspired by quality publications:
|
||||
- **Typography**: DM Sans for body text, Playfair Display for headings
|
||||
- **Color Palette**: Warm cream background with coral and teal accents
|
||||
- **Interactions**: Smooth animations and hover effects
|
||||
- **Charts**: Clean, readable data visualizations
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
# Run with auto-reload
|
||||
python -m uvicorn backend.app:app --reload --port 8000
|
||||
|
||||
# Or run directly
|
||||
python backend/app.py
|
||||
```
|
||||
|
||||
## Coverage
|
||||
|
||||
This application is specifically designed for:
|
||||
|
||||
- **School Phase**: Primary schools only (Key Stage 2)
|
||||
- **Geographic Area**: Wandsworth and Merton (London boroughs)
|
||||
- **Time Period**: Last 5 years of data (2020-2024)
|
||||
|
||||
Note: 2021 data shows as unavailable because SATs were cancelled due to COVID-19.
|
||||
|
||||
## Data Source
|
||||
|
||||
Data is sourced from the UK Government's [Compare School Performance](https://www.compare-school-performance.service.gov.uk/) service, which provides official school performance data for England.
|
||||
|
||||
**Important**: When using real data, please comply with the [terms of use](https://www.compare-school-performance.service.gov.uk/download-data) and data protection regulations.
|
||||
|
||||
## Scheduled Jobs
|
||||
|
||||
### Geocoding Schools (Cron Job)
|
||||
|
||||
School postcodes are geocoded by a scheduled job, not on-demand. This improves performance and reduces API calls.
|
||||
|
||||
**Setup the cron job** (runs weekly on Sunday at 2am):
|
||||
|
||||
```bash
|
||||
# Edit crontab
|
||||
crontab -e
|
||||
|
||||
# Add this line (adjust paths as needed):
|
||||
0 2 * * 0 cd /path/to/school_compare && /path/to/venv/bin/python scripts/geocode_schools.py >> /var/log/geocode_schools.log 2>&1
|
||||
```
|
||||
|
||||
**Manual run:**
|
||||
```bash
|
||||
# Geocode only schools missing coordinates
|
||||
python scripts/geocode_schools.py
|
||||
|
||||
# Force re-geocode all schools
|
||||
python scripts/geocode_schools.py --force
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT License - feel free to use this project for educational purposes.
|
||||
|
||||
---
|
||||
|
||||
Built with ❤️ for Wandsworth & Merton families
|
||||
## Deployment
|
||||
|
||||
Work on a feature branch and open a PR. Merging to `main` builds images and
|
||||
deploys staging. Production promotion is a separate, human-triggered Gitea
|
||||
workflow. Use [DEPLOY.md](docs/DEPLOY.md) and the Portainer compose files as the
|
||||
operational references. The generic compose examples still reference `:latest`,
|
||||
which the current release workflow does not publish.
|
||||
+69
-1
@@ -38,7 +38,7 @@ from .data_loader import (
|
||||
)
|
||||
from .data_loader import get_data_info as get_db_info
|
||||
from . import flags
|
||||
from .places import build_place_registry
|
||||
from .places import build_place_index, build_place_registry, places_for_urn
|
||||
from .schemas import METRIC_DEFINITIONS, RANKING_COLUMNS, SCHOOL_COLUMNS
|
||||
from .utils import clean_for_json, convert_to_native
|
||||
|
||||
@@ -65,6 +65,10 @@ _sitemaps: dict[str, str] | None = None
|
||||
# Built from the same DataFrame the sitemap uses, so places and sitemap can
|
||||
# never describe different corpora. Reset by the same admin endpoint.
|
||||
_place_registry: dict | None = None
|
||||
# Cached beside the registry, and invalidated by identity against it — see
|
||||
# get_place_index. Never cleared independently.
|
||||
_place_index: dict | None = None
|
||||
_place_index_source: dict | None = None
|
||||
|
||||
VALID_PLACE_KINDS = ("town", "locality", "authority", "outcode")
|
||||
|
||||
@@ -188,6 +192,24 @@ def get_place_registry() -> dict:
|
||||
return _place_registry
|
||||
|
||||
|
||||
def get_place_index() -> dict:
|
||||
"""URN → its published places, cached against the registry it came from.
|
||||
|
||||
Invalidation is an identity check rather than a second flag to remember to
|
||||
clear. Anything that drops `_place_registry` — the tests all do — gets a
|
||||
fresh registry object here, which no longer matches the one the index was
|
||||
built from, so the index rebuilds with it. A separate `_place_index = None`
|
||||
would be one more thing to forget, and a stale reverse index is exactly the
|
||||
bug that would put links to another dataset's places on a school page.
|
||||
"""
|
||||
global _place_index, _place_index_source
|
||||
registry = get_place_registry()
|
||||
if _place_index is None or _place_index_source is not registry:
|
||||
_place_index = build_place_index(registry)
|
||||
_place_index_source = registry
|
||||
return _place_index
|
||||
|
||||
|
||||
def _urlset(rows: list[str]) -> str:
|
||||
return "\n".join([
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
@@ -211,6 +233,45 @@ def _place_url(place) -> str:
|
||||
return f"/schools/{place.slug}"
|
||||
|
||||
|
||||
def _places_payload(urn: int) -> list[dict]:
|
||||
"""The published places containing this school, as the school page needs
|
||||
them: a name to write in the link, a count so the anchor can say what it
|
||||
leads to, and the canonical path.
|
||||
|
||||
`phases` carries the phase variants this school actually appears on, which
|
||||
is usually one and is two for an all-through school — it is listed on both
|
||||
pages, so there is no tie to break.
|
||||
|
||||
Membership is read straight from the registry's own `phase_urns` rather
|
||||
than re-derived from the school's phase string. The registry is the one
|
||||
place that decides which phases a place publishes and who is on them;
|
||||
computing it a second time here is how a page comes to link a school to a
|
||||
phase page that does not list it, or to a route that does not exist. That
|
||||
is also why outcodes need no special case: they carry empty `phase_urns`,
|
||||
so they report no phase links on their own.
|
||||
"""
|
||||
payload = []
|
||||
for place in places_for_urn(get_place_index(), int(urn)):
|
||||
phases = [
|
||||
{
|
||||
"phase": phase,
|
||||
"count": len(phase_urns),
|
||||
"url": f"{_place_url(place)}/{phase}",
|
||||
}
|
||||
for phase, phase_urns in sorted(place.phase_urns.items())
|
||||
if int(urn) in phase_urns
|
||||
]
|
||||
payload.append({
|
||||
"kind": place.kind,
|
||||
"slug": place.slug,
|
||||
"name": place.name,
|
||||
"count": len(place.urns),
|
||||
"url": _place_url(place),
|
||||
"phases": phases,
|
||||
})
|
||||
return payload
|
||||
|
||||
|
||||
def _place_sitemap_rows(kinds: tuple[str, ...]) -> list[str]:
|
||||
"""A <url> per place, plus a phase variant wherever that phase clears the
|
||||
threshold on its own.
|
||||
@@ -902,6 +963,13 @@ async def get_school_details(request: Request, urn: int):
|
||||
|
||||
return {
|
||||
"school_info": school_info,
|
||||
# Where this school sits in the location layer, for the page's link
|
||||
# module and breadcrumb. Derived from the same registry the place
|
||||
# pages and the sitemap use, so a link is never offered for a page
|
||||
# that does not exist. Empty is a valid answer: a school whose town
|
||||
# and authority both fall below the publish threshold has nowhere to
|
||||
# point, and the page renders without the module.
|
||||
"places": _places_payload(urn),
|
||||
"yearly_data": clean_for_json(school_data),
|
||||
# Supplementary data (null if not yet populated by Kestra)
|
||||
"ofsted": supplementary.get("ofsted"),
|
||||
|
||||
@@ -188,16 +188,6 @@ def geocode_single_postcode(postcode: str) -> Optional[Tuple[float, float]]:
|
||||
return None
|
||||
|
||||
|
||||
def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
||||
"""Calculate great-circle distance between two points (miles)."""
|
||||
from math import radians, cos, sin, asin, sqrt
|
||||
lat1, lon1, lat2, lon2 = map(radians, [lat1, lon1, lat2, lon2])
|
||||
dlat = lat2 - lat1
|
||||
dlon = lon2 - lon1
|
||||
a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
|
||||
return 2 * asin(sqrt(a)) * 3956
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MAIN DATA LOAD — joins dim_school + dim_location + fact_performance
|
||||
# fact_performance is a merged KS2+KS4 table (one row per URN per year).
|
||||
|
||||
@@ -57,6 +57,23 @@ REGISTRY: dict[str, Flag] = {
|
||||
),
|
||||
added=date(2026, 8, 26),
|
||||
),
|
||||
Flag(
|
||||
name="about_page",
|
||||
description=(
|
||||
"The /about page, its footer link, its sitemap entry, and the "
|
||||
"named-author byline on every blog post."
|
||||
),
|
||||
added=date(2026, 9, 8),
|
||||
),
|
||||
Flag(
|
||||
name="blog",
|
||||
description=(
|
||||
"The /blog index, post pages, the RSS feed, their footer link "
|
||||
"and their sitemap entries. Not /admin: posts must be "
|
||||
"writable before the blog is readable."
|
||||
),
|
||||
added=date(2026, 9, 8),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,512 +0,0 @@
|
||||
"""
|
||||
Database migration logic for importing CSV data.
|
||||
Used by both CLI script and automatic startup migration.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from .config import settings
|
||||
from .database import Base, engine, get_db_session
|
||||
from .models import School, SchoolResult
|
||||
from .schemas import (
|
||||
COLUMN_MAPPINGS,
|
||||
LA_CODE_TO_NAME,
|
||||
NULL_VALUES,
|
||||
SCHOOL_TYPE_MAP,
|
||||
)
|
||||
|
||||
|
||||
def parse_numeric(value) -> Optional[float]:
|
||||
"""Parse a numeric value, handling special cases."""
|
||||
if pd.isna(value):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value) if not np.isnan(value) else None
|
||||
str_val = str(value).strip().upper()
|
||||
if str_val in NULL_VALUES or str_val == "":
|
||||
return None
|
||||
# Remove percentage signs if present
|
||||
str_val = str_val.replace("%", "")
|
||||
try:
|
||||
return float(str_val)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def extract_year_from_folder(folder_name: str) -> Optional[int]:
|
||||
"""Extract year from folder name like '2023-2024'."""
|
||||
match = re.search(r"(\d{4})-(\d{4})", folder_name)
|
||||
if match:
|
||||
return int(match.group(2))
|
||||
match = re.search(r"(\d{4})", folder_name)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def geocode_postcodes_bulk(postcodes: list) -> Dict[str, tuple]:
|
||||
"""
|
||||
Geocode postcodes in bulk using postcodes.io API.
|
||||
Returns dict of postcode -> (latitude, longitude).
|
||||
"""
|
||||
results = {}
|
||||
valid_postcodes = [
|
||||
p.strip().upper()
|
||||
for p in postcodes
|
||||
if p and isinstance(p, str) and len(p.strip()) >= 5
|
||||
]
|
||||
valid_postcodes = list(set(valid_postcodes))
|
||||
|
||||
if not valid_postcodes:
|
||||
return results
|
||||
|
||||
batch_size = 100
|
||||
total_batches = (len(valid_postcodes) + batch_size - 1) // batch_size
|
||||
|
||||
for i, batch_start in enumerate(range(0, len(valid_postcodes), batch_size)):
|
||||
batch = valid_postcodes[batch_start : batch_start + batch_size]
|
||||
print(
|
||||
f" Geocoding batch {i + 1}/{total_batches} ({len(batch)} postcodes)..."
|
||||
)
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
"https://api.postcodes.io/postcodes",
|
||||
json={"postcodes": batch},
|
||||
timeout=30,
|
||||
)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
for item in data.get("result", []):
|
||||
if item and item.get("result"):
|
||||
pc = item["query"].upper()
|
||||
lat = item["result"].get("latitude")
|
||||
lon = item["result"].get("longitude")
|
||||
if lat and lon:
|
||||
results[pc] = (lat, lon)
|
||||
except Exception as e:
|
||||
print(f" Warning: Geocoding batch failed: {e}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def load_csv_data(data_dir: Path) -> pd.DataFrame:
|
||||
"""Load all CSV data from data directory."""
|
||||
all_data = []
|
||||
|
||||
for folder in sorted(data_dir.iterdir()):
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
|
||||
year = extract_year_from_folder(folder.name)
|
||||
if not year:
|
||||
continue
|
||||
|
||||
# Specifically look for the KS2 results file
|
||||
ks2_file = folder / "england_ks2final.csv"
|
||||
if not ks2_file.exists():
|
||||
continue
|
||||
|
||||
csv_file = ks2_file
|
||||
print(f" Loading {csv_file.name} (year {year})...")
|
||||
|
||||
try:
|
||||
df = pd.read_csv(csv_file, encoding="latin-1", low_memory=False)
|
||||
except Exception as e:
|
||||
print(f" Error loading {csv_file}: {e}")
|
||||
continue
|
||||
|
||||
# Rename columns
|
||||
df.rename(columns=COLUMN_MAPPINGS, inplace=True)
|
||||
df["year"] = year
|
||||
|
||||
# Handle local authority name
|
||||
la_name_cols = ["LANAME", "LA (name)", "LA_NAME", "LA NAME"]
|
||||
la_name_col = next((c for c in la_name_cols if c in df.columns), None)
|
||||
|
||||
if la_name_col and la_name_col != "local_authority":
|
||||
df["local_authority"] = df[la_name_col]
|
||||
elif "LEA" in df.columns:
|
||||
df["local_authority_code"] = pd.to_numeric(df["LEA"], errors="coerce")
|
||||
df["local_authority"] = (
|
||||
df["local_authority_code"]
|
||||
.map(LA_CODE_TO_NAME)
|
||||
.fillna(df["LEA"].astype(str))
|
||||
)
|
||||
|
||||
# Store LEA code
|
||||
if "LEA" in df.columns:
|
||||
df["local_authority_code"] = pd.to_numeric(df["LEA"], errors="coerce")
|
||||
|
||||
# Map school type
|
||||
if "school_type_code" in df.columns:
|
||||
df["school_type"] = (
|
||||
df["school_type_code"]
|
||||
.map(SCHOOL_TYPE_MAP)
|
||||
.fillna(df["school_type_code"])
|
||||
)
|
||||
|
||||
# Create combined address
|
||||
addr_parts = ["address1", "address2", "town", "postcode"]
|
||||
for col in addr_parts:
|
||||
if col not in df.columns:
|
||||
df[col] = None
|
||||
|
||||
df["address"] = df.apply(
|
||||
lambda r: ", ".join(
|
||||
str(v)
|
||||
for v in [
|
||||
r.get("address1"),
|
||||
r.get("address2"),
|
||||
r.get("town"),
|
||||
r.get("postcode"),
|
||||
]
|
||||
if pd.notna(v) and str(v).strip()
|
||||
),
|
||||
axis=1,
|
||||
)
|
||||
|
||||
all_data.append(df)
|
||||
print(f" Loaded {len(df)} records")
|
||||
|
||||
if all_data:
|
||||
result = pd.concat(all_data, ignore_index=True)
|
||||
print(f"\nTotal records loaded: {len(result)}")
|
||||
print(f"Unique schools: {result['urn'].nunique()}")
|
||||
print(f"Years: {sorted(result['year'].unique())}")
|
||||
return result
|
||||
|
||||
return pd.DataFrame()
|
||||
|
||||
|
||||
def migrate_data(df: pd.DataFrame, geocode: bool = False, geocode_cache: dict = None):
|
||||
"""Migrate DataFrame data to database."""
|
||||
|
||||
if geocode_cache is None:
|
||||
geocode_cache = {}
|
||||
|
||||
# Clean URN column - convert to integer, drop invalid values
|
||||
df = df.copy()
|
||||
df["urn"] = pd.to_numeric(df["urn"], errors="coerce")
|
||||
df = df.dropna(subset=["urn"])
|
||||
df["urn"] = df["urn"].astype(int)
|
||||
|
||||
# Group by URN to get unique schools (use latest year's data)
|
||||
school_data = (
|
||||
df.sort_values("year", ascending=False).groupby("urn").first().reset_index()
|
||||
)
|
||||
print(f"\nMigrating {len(school_data)} unique schools...")
|
||||
|
||||
# Geocode postcodes that aren't already in the cache
|
||||
geocoded = dict(geocode_cache) # start with preserved coordinates
|
||||
if geocode and "postcode" in df.columns:
|
||||
cached_postcodes = {
|
||||
str(row.get("postcode", "")).strip().upper()
|
||||
for _, row in school_data.iterrows()
|
||||
if int(float(str(row.get("urn", 0) or 0))) in geocode_cache
|
||||
}
|
||||
postcodes_needed = [
|
||||
p for p in df["postcode"].dropna().unique()
|
||||
if str(p).strip().upper() not in cached_postcodes
|
||||
]
|
||||
if postcodes_needed:
|
||||
print(f"\nGeocoding {len(postcodes_needed)} postcodes ({len(geocode_cache)} restored from cache)...")
|
||||
fresh = geocode_postcodes_bulk(postcodes_needed)
|
||||
geocoded.update(fresh)
|
||||
print(f" Successfully geocoded {len(fresh)} new postcodes")
|
||||
else:
|
||||
print(f"\nAll {len(geocode_cache)} postcodes restored from cache, skipping geocoding.")
|
||||
|
||||
with get_db_session() as db:
|
||||
# Create schools
|
||||
urn_to_school_id = {}
|
||||
schools_created = 0
|
||||
|
||||
for _, row in school_data.iterrows():
|
||||
# Safely parse URN - handle None, NaN, whitespace, and invalid values
|
||||
urn_val = row.get("urn")
|
||||
urn = None
|
||||
if pd.notna(urn_val):
|
||||
try:
|
||||
urn_str = str(urn_val).strip()
|
||||
if urn_str:
|
||||
urn = int(float(urn_str)) # Handle "12345.0" format
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if not urn:
|
||||
continue
|
||||
|
||||
# Skip if we've already added this URN (handles duplicates in source data)
|
||||
if urn in urn_to_school_id:
|
||||
continue
|
||||
|
||||
# Get geocoding data
|
||||
postcode = row.get("postcode")
|
||||
lat, lon = None, None
|
||||
if postcode and pd.notna(postcode):
|
||||
coords = geocoded.get(str(postcode).strip().upper())
|
||||
if coords:
|
||||
lat, lon = coords
|
||||
|
||||
# Safely parse local_authority_code
|
||||
la_code = None
|
||||
la_code_val = row.get("local_authority_code")
|
||||
if pd.notna(la_code_val):
|
||||
try:
|
||||
la_code_str = str(la_code_val).strip()
|
||||
if la_code_str:
|
||||
la_code = int(float(la_code_str))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
school = School(
|
||||
urn=urn,
|
||||
school_name=row.get("school_name")
|
||||
if pd.notna(row.get("school_name"))
|
||||
else "Unknown",
|
||||
local_authority=row.get("local_authority")
|
||||
if pd.notna(row.get("local_authority"))
|
||||
else None,
|
||||
local_authority_code=la_code,
|
||||
school_type=row.get("school_type")
|
||||
if pd.notna(row.get("school_type"))
|
||||
else None,
|
||||
school_type_code=row.get("school_type_code")
|
||||
if pd.notna(row.get("school_type_code"))
|
||||
else None,
|
||||
religious_denomination=row.get("religious_denomination")
|
||||
if pd.notna(row.get("religious_denomination"))
|
||||
else None,
|
||||
age_range=row.get("age_range")
|
||||
if pd.notna(row.get("age_range"))
|
||||
else None,
|
||||
address1=row.get("address1") if pd.notna(row.get("address1")) else None,
|
||||
address2=row.get("address2") if pd.notna(row.get("address2")) else None,
|
||||
town=row.get("town") if pd.notna(row.get("town")) else None,
|
||||
postcode=row.get("postcode") if pd.notna(row.get("postcode")) else None,
|
||||
latitude=lat,
|
||||
longitude=lon,
|
||||
)
|
||||
db.add(school)
|
||||
db.flush() # Get the ID
|
||||
urn_to_school_id[urn] = school.id
|
||||
schools_created += 1
|
||||
|
||||
if schools_created % 1000 == 0:
|
||||
print(f" Created {schools_created} schools...")
|
||||
|
||||
print(f" Created {schools_created} schools")
|
||||
|
||||
# Create results
|
||||
print(f"\nMigrating {len(df)} yearly results...")
|
||||
results_created = 0
|
||||
|
||||
for _, row in df.iterrows():
|
||||
# Safely parse URN
|
||||
urn_val = row.get("urn")
|
||||
urn = None
|
||||
if pd.notna(urn_val):
|
||||
try:
|
||||
urn_str = str(urn_val).strip()
|
||||
if urn_str:
|
||||
urn = int(float(urn_str))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if not urn or urn not in urn_to_school_id:
|
||||
continue
|
||||
|
||||
school_id = urn_to_school_id[urn]
|
||||
|
||||
# Safely parse year
|
||||
year_val = row.get("year")
|
||||
year = None
|
||||
if pd.notna(year_val):
|
||||
try:
|
||||
year = int(float(str(year_val).strip()))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if not year:
|
||||
continue
|
||||
|
||||
result = SchoolResult(
|
||||
school_id=school_id,
|
||||
year=year,
|
||||
total_pupils=parse_numeric(row.get("total_pupils")),
|
||||
eligible_pupils=parse_numeric(row.get("eligible_pupils")),
|
||||
# Expected Standard
|
||||
rwm_expected_pct=parse_numeric(row.get("rwm_expected_pct")),
|
||||
reading_expected_pct=parse_numeric(row.get("reading_expected_pct")),
|
||||
writing_expected_pct=parse_numeric(row.get("writing_expected_pct")),
|
||||
maths_expected_pct=parse_numeric(row.get("maths_expected_pct")),
|
||||
gps_expected_pct=parse_numeric(row.get("gps_expected_pct")),
|
||||
science_expected_pct=parse_numeric(row.get("science_expected_pct")),
|
||||
# Higher Standard
|
||||
rwm_high_pct=parse_numeric(row.get("rwm_high_pct")),
|
||||
reading_high_pct=parse_numeric(row.get("reading_high_pct")),
|
||||
writing_high_pct=parse_numeric(row.get("writing_high_pct")),
|
||||
maths_high_pct=parse_numeric(row.get("maths_high_pct")),
|
||||
gps_high_pct=parse_numeric(row.get("gps_high_pct")),
|
||||
# Progress
|
||||
reading_progress=parse_numeric(row.get("reading_progress")),
|
||||
writing_progress=parse_numeric(row.get("writing_progress")),
|
||||
maths_progress=parse_numeric(row.get("maths_progress")),
|
||||
# Averages
|
||||
reading_avg_score=parse_numeric(row.get("reading_avg_score")),
|
||||
maths_avg_score=parse_numeric(row.get("maths_avg_score")),
|
||||
gps_avg_score=parse_numeric(row.get("gps_avg_score")),
|
||||
# Context
|
||||
disadvantaged_pct=parse_numeric(row.get("disadvantaged_pct")),
|
||||
eal_pct=parse_numeric(row.get("eal_pct")),
|
||||
sen_support_pct=parse_numeric(row.get("sen_support_pct")),
|
||||
sen_ehcp_pct=parse_numeric(row.get("sen_ehcp_pct")),
|
||||
stability_pct=parse_numeric(row.get("stability_pct")),
|
||||
# Absence
|
||||
reading_absence_pct=parse_numeric(row.get("reading_absence_pct")),
|
||||
gps_absence_pct=parse_numeric(row.get("gps_absence_pct")),
|
||||
maths_absence_pct=parse_numeric(row.get("maths_absence_pct")),
|
||||
writing_absence_pct=parse_numeric(row.get("writing_absence_pct")),
|
||||
science_absence_pct=parse_numeric(row.get("science_absence_pct")),
|
||||
# Gender
|
||||
rwm_expected_boys_pct=parse_numeric(row.get("rwm_expected_boys_pct")),
|
||||
rwm_expected_girls_pct=parse_numeric(row.get("rwm_expected_girls_pct")),
|
||||
rwm_high_boys_pct=parse_numeric(row.get("rwm_high_boys_pct")),
|
||||
rwm_high_girls_pct=parse_numeric(row.get("rwm_high_girls_pct")),
|
||||
# Disadvantaged
|
||||
rwm_expected_disadvantaged_pct=parse_numeric(
|
||||
row.get("rwm_expected_disadvantaged_pct")
|
||||
),
|
||||
rwm_expected_non_disadvantaged_pct=parse_numeric(
|
||||
row.get("rwm_expected_non_disadvantaged_pct")
|
||||
),
|
||||
disadvantaged_gap=parse_numeric(row.get("disadvantaged_gap")),
|
||||
# 3-Year
|
||||
rwm_expected_3yr_pct=parse_numeric(row.get("rwm_expected_3yr_pct")),
|
||||
reading_avg_3yr=parse_numeric(row.get("reading_avg_3yr")),
|
||||
maths_avg_3yr=parse_numeric(row.get("maths_avg_3yr")),
|
||||
)
|
||||
db.add(result)
|
||||
results_created += 1
|
||||
|
||||
if results_created % 10000 == 0:
|
||||
print(f" Created {results_created} results...")
|
||||
db.flush()
|
||||
|
||||
print(f" Created {results_created} results")
|
||||
|
||||
# Commit all changes
|
||||
db.commit()
|
||||
print("\nMigration complete!")
|
||||
|
||||
|
||||
def _apply_schema_alterations():
|
||||
"""
|
||||
Add new columns to existing tables using ALTER TABLE … ADD COLUMN IF NOT EXISTS.
|
||||
Safe to run on every migration — no-ops if the column already exists.
|
||||
Add entries here whenever models.py gains new columns on an existing table.
|
||||
"""
|
||||
alterations = [
|
||||
# v4: Ofsted Report Card columns
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS framework VARCHAR(20)",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_safeguarding_met BOOLEAN",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_inclusion INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_curriculum_teaching INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_achievement INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_attendance_behaviour INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_personal_development INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_leadership_governance INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_early_years INTEGER",
|
||||
"ALTER TABLE ofsted_inspections ADD COLUMN IF NOT EXISTS rc_sixth_form INTEGER",
|
||||
]
|
||||
from sqlalchemy import text as sa_text
|
||||
with engine.connect() as conn:
|
||||
for stmt in alterations:
|
||||
try:
|
||||
conn.execute(sa_text(stmt))
|
||||
except Exception as e:
|
||||
print(f" Warning: alteration skipped ({e})")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _apply_schema_drops():
|
||||
"""
|
||||
Drop tables retired from the schema. Idempotent (DROP … IF EXISTS), so it's
|
||||
safe to run on every migration. Add entries here when a model is removed.
|
||||
"""
|
||||
drops = [
|
||||
# v6: Ofsted Parent View feature removed
|
||||
"DROP TABLE IF EXISTS marts.fact_parent_view CASCADE",
|
||||
]
|
||||
from sqlalchemy import text as sa_text
|
||||
with engine.connect() as conn:
|
||||
for stmt in drops:
|
||||
try:
|
||||
conn.execute(sa_text(stmt))
|
||||
except Exception as e:
|
||||
print(f" Warning: drop skipped ({e})")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def run_full_migration(geocode: bool = False) -> bool:
|
||||
"""
|
||||
Run a complete migration: drop all tables and reimport from CSV.
|
||||
|
||||
Returns True if successful, False if no data found.
|
||||
Raises exception on error.
|
||||
"""
|
||||
# Preserve existing geocoding so a reimport doesn't throw away coordinates
|
||||
# that took a long time to compute.
|
||||
geocode_cache: dict[int, tuple[float, float]] = {}
|
||||
inspector = __import__("sqlalchemy").inspect(engine)
|
||||
if "schools" in inspector.get_table_names():
|
||||
try:
|
||||
with get_db_session() as db:
|
||||
rows = db.execute(
|
||||
__import__("sqlalchemy").text(
|
||||
"SELECT urn, latitude, longitude FROM schools "
|
||||
"WHERE latitude IS NOT NULL AND longitude IS NOT NULL"
|
||||
)
|
||||
).fetchall()
|
||||
geocode_cache = {r.urn: (r.latitude, r.longitude) for r in rows}
|
||||
print(f" Saved {len(geocode_cache)} existing geocoded coordinates.")
|
||||
except Exception as e:
|
||||
print(f" Warning: could not save geocode cache: {e}")
|
||||
|
||||
# Only drop the core KS2 tables — leave supplementary tables (ofsted, census,
|
||||
# finance, etc.) intact so a reimport doesn't wipe integrator-populated data.
|
||||
# schema_version is NOT dropped: it persists so restarts don't re-trigger migration.
|
||||
ks2_tables = ["school_results", "schools"]
|
||||
print(f"Dropping core tables: {ks2_tables} ...")
|
||||
inspector = __import__("sqlalchemy").inspect(engine)
|
||||
existing = set(inspector.get_table_names())
|
||||
for tname in ks2_tables:
|
||||
if tname in existing:
|
||||
Base.metadata.tables[tname].drop(bind=engine)
|
||||
|
||||
print("Creating all tables...")
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
# ALTER existing supplementary tables to add any new columns.
|
||||
# create_all() only creates missing tables; it won't add columns to tables
|
||||
# that already exist from an older schema version. These statements are
|
||||
# idempotent (IF NOT EXISTS) so they're safe to run on every migration.
|
||||
print("Applying column additions to supplementary tables...")
|
||||
_apply_schema_alterations()
|
||||
|
||||
print("Dropping retired tables...")
|
||||
_apply_schema_drops()
|
||||
|
||||
print("\nLoading CSV data...")
|
||||
df = load_csv_data(settings.data_dir)
|
||||
|
||||
if df.empty:
|
||||
print("Warning: No CSV data found to migrate!")
|
||||
return False
|
||||
|
||||
migrate_data(df, geocode=geocode, geocode_cache=geocode_cache)
|
||||
return True
|
||||
@@ -296,6 +296,48 @@ def _locality_places(df, publishable: set[int],
|
||||
return out
|
||||
|
||||
|
||||
# Ordered authority → town/locality → outcode, widest first, because that is
|
||||
# the order a breadcrumb reads. The link module re-sorts for its own purposes.
|
||||
_PLACE_ORDER = {"authority": 0, "town": 1, "locality": 2, "outcode": 3}
|
||||
|
||||
|
||||
def build_place_index(registry: dict[str, Place]) -> dict[int, tuple[Place, ...]]:
|
||||
"""URN → the published places containing it, built once per registry.
|
||||
|
||||
The reverse of the registry, and the thing school pages link out through.
|
||||
Derived from the registry rather than maintained beside it, so the two
|
||||
cannot disagree about which places exist: a place below the publish
|
||||
threshold is absent from the registry, so it is absent from here too, and
|
||||
a link is never offered for a page that does not exist.
|
||||
|
||||
Built as an index rather than scanned per call because /api/schools/{urn}
|
||||
is the site's highest-traffic endpoint. Scanning meant walking every place
|
||||
and doing a tuple membership test against each — on the order of 10^5
|
||||
comparisons per request, repeated for every school page view. One pass at
|
||||
registry-build time replaces all of it with a dict lookup.
|
||||
"""
|
||||
grouped: dict[int, list[Place]] = {}
|
||||
for place in registry.values():
|
||||
for urn in place.urns:
|
||||
grouped.setdefault(int(urn), []).append(place)
|
||||
|
||||
return {
|
||||
urn: tuple(sorted(places,
|
||||
key=lambda p: (_PLACE_ORDER.get(p.kind, 9), p.slug)))
|
||||
for urn, places in grouped.items()
|
||||
}
|
||||
|
||||
|
||||
def places_for_urn(index: dict[int, tuple[Place, ...]], urn: int) -> tuple[Place, ...]:
|
||||
"""The published places containing this school, widest first.
|
||||
|
||||
Empty is a real answer, not a failure: a school whose town and authority
|
||||
both fall below the publish threshold has nowhere to link, and the page
|
||||
renders without the module.
|
||||
"""
|
||||
return index.get(int(urn), ())
|
||||
|
||||
|
||||
def build_place_registry(df) -> dict[str, Place]:
|
||||
"""Every place the site publishes, keyed by "<kind>:<slug>"."""
|
||||
if df.empty or "urn" not in df.columns:
|
||||
|
||||
@@ -8,7 +8,8 @@ import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from backend.places import MIN_SCHOOLS, build_place_registry
|
||||
from backend.places import (MIN_SCHOOLS, build_place_index,
|
||||
build_place_registry, places_for_urn)
|
||||
|
||||
|
||||
def _df(rows: list[dict]) -> pd.DataFrame:
|
||||
@@ -418,3 +419,79 @@ def test_an_authority_still_publishes_phase_variants():
|
||||
and /schools/authority/[la]/[phase] is the route that serves it."""
|
||||
reg = build_place_registry(_df(_town(MIN_SCHOOLS, "Maidstone", "Kent")))
|
||||
assert reg["authority:kent"].publishes_phase("primary")
|
||||
|
||||
|
||||
# ── The reverse index: which published places contain a school ──────────────
|
||||
#
|
||||
# School pages link out to the location layer through this. It is the whole
|
||||
# point of the index: before it, ~27k school pages linked to nothing on the
|
||||
# site and stranded whatever authority they held.
|
||||
|
||||
def test_a_school_resolves_to_every_published_place_containing_it():
|
||||
reg = build_place_registry(_df(_town(MIN_SCHOOLS, "Brentwood", "Essex")))
|
||||
places = places_for_urn(build_place_index(reg), 100000)
|
||||
|
||||
kinds = {p.kind for p in places}
|
||||
assert "town" in kinds
|
||||
assert "authority" in kinds
|
||||
|
||||
|
||||
def test_a_school_in_an_unpublished_town_still_resolves_to_its_authority():
|
||||
# A town below the threshold has no page, so there is no link to offer —
|
||||
# but the authority above it clears the threshold on the same schools and
|
||||
# is where that reader should be sent.
|
||||
reg = build_place_registry(_df(
|
||||
_town(MIN_SCHOOLS - 1, "Tinytown", "Essex")
|
||||
+ _town(MIN_SCHOOLS, "Brentwood", "Essex", start=200000)
|
||||
))
|
||||
places = places_for_urn(build_place_index(reg), 100000)
|
||||
|
||||
# The town is below the threshold, so it has no page and must not be
|
||||
# offered as a link. The authority above it does, and is the right target.
|
||||
assert all(p.slug != "tinytown" for p in places)
|
||||
assert "authority" in {p.kind for p in places}
|
||||
|
||||
|
||||
def test_an_unknown_urn_resolves_to_nothing_rather_than_raising():
|
||||
# A school page renders for any URN the API knows; the link module is not
|
||||
# entitled to take the page down when it has nothing to say.
|
||||
reg = build_place_registry(_df(_town(MIN_SCHOOLS, "Brentwood", "Essex")))
|
||||
assert places_for_urn(build_place_index(reg), 999999) == ()
|
||||
|
||||
|
||||
def test_the_index_is_consistent_with_the_registry_it_was_built_from():
|
||||
# The invariant that matters: a link module must never offer a place whose
|
||||
# page does not exist, and never omit one that does.
|
||||
reg = build_place_registry(_df(
|
||||
_town(MIN_SCHOOLS, "Brentwood", "Essex")
|
||||
+ _town(MIN_SCHOOLS, "Bedford", "Bedford", start=300000)
|
||||
))
|
||||
index = build_place_index(reg)
|
||||
for key, place in reg.items():
|
||||
for urn in place.urns:
|
||||
assert place in places_for_urn(index, urn), (
|
||||
f"{urn} is in {key} but the index does not say so")
|
||||
|
||||
|
||||
def test_the_index_holds_no_school_the_registry_does_not():
|
||||
# The reverse direction of the invariant above. An index entry for a URN
|
||||
# no published place contains would put a link on a page for a place that
|
||||
# does not list that school.
|
||||
reg = build_place_registry(_df(
|
||||
_town(MIN_SCHOOLS, "Brentwood", "Essex")
|
||||
+ _town(MIN_SCHOOLS - 1, "Tinytown", "Essex", start=400000)
|
||||
))
|
||||
index = build_place_index(reg)
|
||||
|
||||
for urn, places in index.items():
|
||||
for place in places:
|
||||
assert urn in place.urns
|
||||
assert place.key in reg
|
||||
|
||||
|
||||
def test_the_index_preserves_the_widest_first_order():
|
||||
# The breadcrumb reads authority then town, and takes this order as given.
|
||||
reg = build_place_registry(_df(_town(MIN_SCHOOLS, "Brentwood", "Essex")))
|
||||
kinds = [p.kind for p in places_for_urn(build_place_index(reg), 100000)]
|
||||
|
||||
assert kinds.index("authority") < kinds.index("town")
|
||||
@@ -56,6 +56,11 @@ def client(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
app_module, "get_supplementary_data", lambda db, urn: {}
|
||||
)
|
||||
# The place registry is a module-level cache, so without this the endpoint
|
||||
# answers from whatever registry an earlier test happened to leave behind
|
||||
# — and a `places == []` assertion is satisfied by a stale registry just
|
||||
# as well as by this fixture's own data, which makes it prove nothing.
|
||||
monkeypatch.setattr(app_module, "_place_registry", None)
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@@ -69,3 +74,174 @@ def test_nan_gias_fields_serialize_as_null(client):
|
||||
assert info["capacity"] is None
|
||||
assert info["total_pupils"] is None
|
||||
assert info["school_name"] == "West London Performing Arts Academy"
|
||||
|
||||
|
||||
# ── Links out to the location layer ─────────────────────────────────────────
|
||||
#
|
||||
# School pages carried no link into the site at all: the only anchor on the
|
||||
# template pointed at the school's own website, so ~27k pages received
|
||||
# whatever authority the site had and sent it off-site. `places` is what the
|
||||
# link module and the breadcrumb are built from.
|
||||
|
||||
def test_places_is_present_even_when_the_school_belongs_to_none(client):
|
||||
# This fixture's single school cannot clear any publish threshold, so the
|
||||
# honest answer is an empty list. The key must still be there: a missing
|
||||
# key and "no places" are different things to the page rendering it.
|
||||
body = client.get("/api/schools/150275").json()
|
||||
assert body["places"] == []
|
||||
|
||||
|
||||
def test_places_names_only_pages_that_exist(monkeypatch):
|
||||
from backend import app as app_module
|
||||
from backend.places import MIN_SCHOOLS
|
||||
|
||||
def _df():
|
||||
return pd.DataFrame([
|
||||
{
|
||||
"urn": 100000 + i,
|
||||
"school_name": f"Brentwood School {i}",
|
||||
"town": "Brentwood",
|
||||
"local_authority": "Essex",
|
||||
"postcode": "CM15 8AA",
|
||||
"phase": "Primary",
|
||||
"year": 202425,
|
||||
"rwm_expected_pct": 60.0,
|
||||
"attainment_8_score": np.nan,
|
||||
"ofsted_grade": 2.0,
|
||||
"ofsted_date": None,
|
||||
}
|
||||
for i in range(MIN_SCHOOLS)
|
||||
])
|
||||
|
||||
monkeypatch.setattr(app_module, "load_school_data", _df)
|
||||
monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
|
||||
monkeypatch.setattr(app_module, "_place_registry", None)
|
||||
client = TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
places = client.get("/api/schools/100000").json()["places"]
|
||||
assert places, "a school in a published town must offer links"
|
||||
|
||||
by_kind = {p["kind"]: p for p in places}
|
||||
assert by_kind["town"]["url"] == "/schools/brentwood"
|
||||
assert by_kind["authority"]["url"] == "/schools/authority/essex"
|
||||
|
||||
# Every entry carries what the link text needs, and a count, so the anchor
|
||||
# can say what it leads to rather than "click here".
|
||||
for place in places:
|
||||
assert place["name"]
|
||||
assert place["count"] >= 1
|
||||
assert place["url"].startswith("/schools/")
|
||||
|
||||
|
||||
def _brentwood_df(phase: str = "Primary", n: int = None):
|
||||
from backend.places import MIN_SCHOOLS
|
||||
n = n if n is not None else MIN_SCHOOLS
|
||||
return lambda: pd.DataFrame([
|
||||
{
|
||||
"urn": 100000 + i,
|
||||
"school_name": f"Brentwood School {i}",
|
||||
"town": "Brentwood", "local_authority": "Essex",
|
||||
"postcode": "CM15 8AA", "phase": phase, "year": 202425,
|
||||
"rwm_expected_pct": 60.0, "attainment_8_score": 50.0,
|
||||
"ofsted_grade": 2.0, "ofsted_date": None,
|
||||
}
|
||||
for i in range(n)
|
||||
])
|
||||
|
||||
|
||||
def _places_for(monkeypatch, df_factory, urn: int):
|
||||
from backend import app as app_module
|
||||
monkeypatch.setattr(app_module, "load_school_data", df_factory)
|
||||
monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
|
||||
monkeypatch.setattr(app_module, "_place_registry", None)
|
||||
client = TestClient(app_module.app, raise_server_exceptions=False)
|
||||
return client.get(f"/api/schools/{urn}").json()["places"]
|
||||
|
||||
|
||||
def test_a_place_offers_the_phase_page_this_school_appears_on(monkeypatch):
|
||||
# "primary schools in brentwood" is the query the phase pages exist for,
|
||||
# and ~950 of them were once reachable by nothing at all.
|
||||
places = _places_for(monkeypatch, _brentwood_df("Primary"), 100000)
|
||||
town = next(p for p in places if p["kind"] == "town")
|
||||
|
||||
assert town["phases"], "a primary school in a published primary town has a link"
|
||||
assert town["phases"][0]["url"] == "/schools/brentwood/primary"
|
||||
assert town["phases"][0]["count"] >= 1
|
||||
|
||||
|
||||
def test_an_all_through_school_offers_both_phase_pages(monkeypatch):
|
||||
# It genuinely appears on both, so there is no tie to break.
|
||||
places = _places_for(monkeypatch, _brentwood_df("All-through"), 100000)
|
||||
town = next(p for p in places if p["kind"] == "town")
|
||||
|
||||
assert {p["phase"] for p in town["phases"]} == {"primary", "secondary"}
|
||||
|
||||
|
||||
def test_outcodes_never_offer_a_phase_page(monkeypatch):
|
||||
# The registry gives outcodes no phase route — nobody searches "primary
|
||||
# schools in SW11" — and computing them anyway once put a link to a
|
||||
# nonexistent route on all 1,720 outcode pages.
|
||||
places = _places_for(monkeypatch, _brentwood_df("Primary"), 100000)
|
||||
outcode = next((p for p in places if p["kind"] == "outcode"), None)
|
||||
|
||||
if outcode is not None:
|
||||
assert outcode["phases"] == []
|
||||
|
||||
|
||||
def test_a_school_absent_from_the_phase_page_is_not_linked_to_it(monkeypatch):
|
||||
# The check is URN membership in the registry's own phase list, not a
|
||||
# re-derivation of the phase mapping. A secondary school must not be sent
|
||||
# to a primary phase page that does not list it.
|
||||
from backend.places import MIN_SCHOOLS
|
||||
|
||||
def df():
|
||||
rows = [
|
||||
{"urn": 100000 + i, "school_name": f"P{i}", "town": "Brentwood",
|
||||
"local_authority": "Essex", "postcode": "CM15 8AA",
|
||||
"phase": "Primary", "year": 202425, "rwm_expected_pct": 60.0,
|
||||
"attainment_8_score": np.nan, "ofsted_grade": 2.0,
|
||||
"ofsted_date": None}
|
||||
for i in range(MIN_SCHOOLS)
|
||||
]
|
||||
rows.append({
|
||||
"urn": 900000, "school_name": "Lone Secondary", "town": "Brentwood",
|
||||
"local_authority": "Essex", "postcode": "CM15 8AA",
|
||||
"phase": "Secondary", "year": 202425, "rwm_expected_pct": np.nan,
|
||||
"attainment_8_score": 50.0, "ofsted_grade": 2.0, "ofsted_date": None,
|
||||
})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
places = _places_for(monkeypatch, df, 900000)
|
||||
town = next(p for p in places if p["kind"] == "town")
|
||||
|
||||
# The town publishes a primary page, but this secondary school is not on
|
||||
# it, and there are too few secondaries for a secondary page.
|
||||
assert town["phases"] == []
|
||||
|
||||
|
||||
def test_the_place_index_rebuilds_when_the_registry_is_replaced(monkeypatch):
|
||||
"""The reverse index is cached; a stale one would put another dataset's
|
||||
places on a school page. Invalidation is an identity check against the
|
||||
registry rather than a second flag, so this asserts the check works."""
|
||||
from backend import app as app_module
|
||||
|
||||
monkeypatch.setattr(app_module, "_place_registry", None)
|
||||
monkeypatch.setattr(app_module, "_place_index", None)
|
||||
monkeypatch.setattr(app_module, "_place_index_source", None)
|
||||
monkeypatch.setattr(app_module, "load_school_data", _brentwood_df("Primary"))
|
||||
|
||||
first = app_module.get_place_index()
|
||||
assert 100000 in first
|
||||
|
||||
# Same registry object, so the index is reused rather than rebuilt.
|
||||
assert app_module.get_place_index() is first
|
||||
|
||||
# Drop the registry the way every test that touches place data does. The
|
||||
# index must follow it, not survive it.
|
||||
app_module._place_registry = None
|
||||
monkeypatch.setattr(app_module, "load_school_data",
|
||||
_brentwood_df("Primary", n=0))
|
||||
|
||||
rebuilt = app_module.get_place_index()
|
||||
assert rebuilt is not first
|
||||
assert 100000 not in rebuilt, "the index outlived the registry it came from"
|
||||
@@ -1,26 +0,0 @@
|
||||
"""
|
||||
Schema versioning for database migrations.
|
||||
|
||||
HOW TO USE:
|
||||
- Bump SCHEMA_VERSION when making changes to database models
|
||||
- This triggers an automatic full data reimport on next app startup
|
||||
|
||||
WHEN TO BUMP:
|
||||
- Adding/removing columns in models.py
|
||||
- Changing column types or constraints
|
||||
- Modifying CSV column mappings in schemas.py
|
||||
- Any change that requires fresh data import
|
||||
"""
|
||||
|
||||
# Current schema version - increment when models change
|
||||
SCHEMA_VERSION = 6
|
||||
|
||||
# Changelog for documentation
|
||||
SCHEMA_CHANGELOG = {
|
||||
1: "Initial schema with School and SchoolResult tables",
|
||||
2: "Added pupil absence fields (reading, maths, gps, writing, science)",
|
||||
3: "Added supplementary data tables: ofsted, parent_view, census, admissions, sen_detail, phonics, deprivation, finance; GIAS columns on schools",
|
||||
4: "Added Ofsted Report Card columns to ofsted_inspections (new framework from Nov 2025)",
|
||||
5: "Apply ALTER TABLE additions for RC columns missed by create_all on existing tables",
|
||||
6: "Removed the Ofsted Parent View feature: dropped fact_parent_view table and model",
|
||||
}
|
||||
@@ -1,180 +1,37 @@
|
||||
# SchoolCompare.co.uk - Project Context
|
||||
# SchoolCompare project context
|
||||
|
||||
## Overview
|
||||
## Maintained documentation
|
||||
|
||||
SchoolCompare is a web application for comparing UK primary school (KS2) performance data. It allows users to:
|
||||
- Search and browse schools by name, location (postcode), or local authority
|
||||
- Compare multiple schools side-by-side with charts and tables
|
||||
- View school rankings by various KS2 metrics
|
||||
- See historical performance trends across years
|
||||
Read [README.md](README.md), [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) and
|
||||
[docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) for the current implementation.
|
||||
[docs/LEGACY_CODE.md](docs/LEGACY_CODE.md) records obsolete paths and deliberate
|
||||
compatibility code. Historical design documents are not current setup instructions.
|
||||
|
||||
## Architecture
|
||||
## Architecture constraints
|
||||
|
||||
### Backend (Python/FastAPI)
|
||||
- **Framework**: FastAPI with uvicorn
|
||||
- **Database**: PostgreSQL with SQLAlchemy ORM
|
||||
- **Data Source**: UK Government "Compare School Performance" CSV downloads
|
||||
|
||||
Key files:
|
||||
- `backend/app.py` - Main FastAPI application, API routes
|
||||
- `backend/config.py` - Configuration via pydantic-settings (env vars, .env file)
|
||||
- `backend/database.py` - SQLAlchemy engine, session management
|
||||
- `backend/models.py` - Database models (School, SchoolResult)
|
||||
- `backend/data_loader.py` - Data queries, geocoding, legacy DataFrame compatibility
|
||||
- `backend/schemas.py` - Column mappings, metric definitions, LA code mappings
|
||||
|
||||
### Content / CMS (Payload)
|
||||
|
||||
Payload CMS runs **inside** the Next.js app — one image, one container, no
|
||||
separate service. It powers `/blog`; `/about` is a plain coded page.
|
||||
|
||||
- **Admin panel:** `/admin`. The only authenticated surface on the site.
|
||||
`noindex` via both `robots.txt` and `X-Robots-Tag`.
|
||||
- **CMS API:** `/cms-api`, **not** `/api`. `/api/*` is a catch-all proxy to
|
||||
FastAPI (`app/(frontend)/api/[...path]`) which would silently swallow every
|
||||
admin call and forward it to the backend. Mount points are defined once in
|
||||
`lib/payloadRoutes.ts`.
|
||||
- **Database:** the existing Postgres, in its own `payload` schema, so no
|
||||
pipeline operation on `public` — including
|
||||
`scripts/migrate_csv_to_db.py --drop` — can reach blog content.
|
||||
- **Uploads:** the `payload_media` Docker volume at `/app/media`. Not
|
||||
reproducible from the pipeline; must be backed up.
|
||||
- **New env vars:** `DATABASE_URL` and `PAYLOAD_SECRET` on the frontend service.
|
||||
Staging must use a different `PAYLOAD_SECRET` from production.
|
||||
- Publishing workflow and house style: `nextjs-app/docs/PUBLISHING.md`.
|
||||
- **Admin field components resolve through a generated import map**
|
||||
(`app/(payload)/admin/importMap.js`). Payload hands the client a *path* per
|
||||
field and looks it up there; a missing entry renders no field and reports no
|
||||
error, while `required` still blocks the save. After adding or changing any
|
||||
field, editor or lexical feature, run `npm run generate:importmap` in
|
||||
`nextjs-app/` and commit the result.
|
||||
|
||||
### Two route groups
|
||||
|
||||
`nextjs-app/app/` has no root `layout.tsx`. It cannot: Payload's admin panel
|
||||
ships its own root layout rendering `<html>`/`<body>`, and Next permits
|
||||
multiple root layouts only when no `app/layout.tsx` exists.
|
||||
|
||||
- `app/(frontend)/` — the site. Its `layout.tsx` is the site's root layout.
|
||||
- `app/(payload)/` — the admin panel and `/cms-api`.
|
||||
|
||||
Route groups are invisible to routing, so every public URL is unchanged.
|
||||
|
||||
**The metadata file conventions stay at the `app/` root** — `robots.ts`,
|
||||
`opengraph-image.tsx`, `icon.png`, `apple-icon.png`. Inside a route group Next
|
||||
treats them as segment-scoped: it renames `/icon.png` to `/icon-<hash>.png` and
|
||||
drops `/robots.txt` entirely. Route handlers are unaffected.
|
||||
|
||||
The build must succeed with `DATABASE_URL` unset, because CI builds it that
|
||||
way. Never call `getCachedPayload()` at module scope, and never add
|
||||
`generateStaticParams` to a DB-backed route.
|
||||
|
||||
### Frontend (Vanilla JS)
|
||||
- Single-page application with hash-based routing
|
||||
- Chart.js for data visualization
|
||||
- No build step required
|
||||
|
||||
Key files:
|
||||
- `frontend/index.html` - Main HTML structure
|
||||
- `frontend/app.js` - All application logic, API calls, rendering
|
||||
- `frontend/styles.css` - Styling (CSS variables, responsive design)
|
||||
|
||||
### Database Schema
|
||||
|
||||
```
|
||||
schools school_results
|
||||
├── id (PK) ├── id (PK)
|
||||
├── urn (unique, indexed) ├── school_id (FK → schools.id)
|
||||
├── school_name ├── year (indexed)
|
||||
├── local_authority ├── rwm_expected_pct
|
||||
├── school_type ├── reading_expected_pct
|
||||
├── postcode ├── ... (all KS2 metrics)
|
||||
├── latitude, longitude └── unique(school_id, year)
|
||||
└── results → SchoolResult[]
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables (or `.env` file):
|
||||
- `DATABASE_URL` - PostgreSQL connection string (default: `postgresql://schoolcompare:schoolcompare@localhost:5432/schoolcompare`)
|
||||
- `HOST`, `PORT` - Server binding (default: `0.0.0.0:80`)
|
||||
- `ALLOWED_ORIGINS` - CORS origins
|
||||
|
||||
## Running Locally
|
||||
|
||||
1. Start PostgreSQL:
|
||||
```bash
|
||||
docker compose up -d db
|
||||
```
|
||||
|
||||
2. Run migration to import CSV data:
|
||||
```bash
|
||||
python scripts/migrate_csv_to_db.py --drop
|
||||
# Add --geocode to geocode postcodes (slower, adds lat/long)
|
||||
```
|
||||
|
||||
3. Start the app:
|
||||
```bash
|
||||
uvicorn backend.app:app --host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
## Docker Deployment
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
This starts:
|
||||
- `db` - PostgreSQL 16 with persistent volume
|
||||
- `app` - FastAPI application on port 80
|
||||
|
||||
## Data
|
||||
|
||||
- Source: UK Government Compare School Performance downloads
|
||||
- Location: `data/` directory with year folders (e.g., `2023-2024/england_ks2final.csv`)
|
||||
- The `scripts/download_data.py` can fetch data from the government website
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Location Search**: Enter postcode to find nearby schools (uses postcodes.io API)
|
||||
- **Multi-school Comparison**: Select multiple schools, view metrics across years
|
||||
- **Rankings**: Top schools by any KS2 metric, filterable by local authority
|
||||
- **Variability Analysis**: Shows standard deviation of scores across years
|
||||
|
||||
## API Endpoints
|
||||
|
||||
- `GET /api/schools` - List/search schools (supports pagination, location search)
|
||||
- `GET /api/schools/{urn}` - School details with all yearly data
|
||||
- `GET /api/compare?urns=123,456` - Compare multiple schools
|
||||
- `GET /api/rankings` - School rankings by metric
|
||||
- `GET /api/filters` - Available filter options (LAs, types, years)
|
||||
- `GET /api/metrics` - Metric definitions (single source of truth)
|
||||
- `GET /api/data-info` - Database stats
|
||||
- Next.js serves the public UI. FastAPI serves school data from dbt-built `marts.*`.
|
||||
The backend does not create school tables or import CSVs at startup.
|
||||
- School coverage spans England and multiple phases, not only primary schools in
|
||||
Wandsworth and Merton.
|
||||
- `/api/*` belongs to the FastAPI proxy. Payload uses `/cms-api` and `/admin`.
|
||||
- Payload runs inside Next.js, with its own `payload` schema and persistent media.
|
||||
Keep CMS migrations independent of school-data transformations.
|
||||
- Public and Payload route groups have separate root layouts. Do not introduce
|
||||
`app/layout.tsx`. Keep site-wide metadata files at the `app/` root.
|
||||
- Builds must succeed with `DATABASE_URL` unset. Do not call `getCachedPayload()`
|
||||
at module scope or add DB-backed `generateStaticParams`.
|
||||
- After changing CMS fields/editors, run `npm run generate:importmap` and commit
|
||||
the generated import map. See `nextjs-app/docs/PUBLISHING.md`.
|
||||
- The backend and pipeline GIAS dictionary copies are generated together; preserve
|
||||
their parity. Tests enforce it.
|
||||
|
||||
## SDLC
|
||||
|
||||
Full details in `docs/DEPLOY.md`. The short version:
|
||||
Follow [docs/DEPLOY.md](docs/DEPLOY.md).
|
||||
|
||||
- **Never push to `main` directly.** Work on a feature branch and open a PR;
|
||||
branch protection requires the PR checks (typecheck, tests, builds, AI review)
|
||||
to pass before merge.
|
||||
- Merging to `main` deploys automatically **to staging only**: images are
|
||||
built once, deployed to the staging Portainer stack, and verified by the
|
||||
Playwright journeys in `e2e/`. Production is a second, manual approval:
|
||||
the "Promote to Production (manual)" workflow in Gitea Actions, run after
|
||||
testing the feature on staging. It refuses commits whose staging E2E gate
|
||||
isn't green. Never trigger it yourself — promotion is the human's call.
|
||||
- If you change user-facing behaviour, update or extend the `e2e/` journey
|
||||
tests in the same PR — they gate whether staging is fit for human testing
|
||||
and whether a commit is promotable.
|
||||
|
||||
## Recent Changes
|
||||
|
||||
- Added staging environment + automated staging→prod pipeline (Gitea Actions)
|
||||
- Migrated from CSV file storage to PostgreSQL database
|
||||
- Added location-based search using postcode geocoding
|
||||
- Added local authority filter to rankings
|
||||
- Improved frontend with featured schools, loading states, API caching
|
||||
|
||||
# Important
|
||||
- Do not attempt to start a local server to test the application, it does not work
|
||||
- Never push directly to `main`. Use a feature branch and a PR with passing checks.
|
||||
- Merges deploy staging only. Production promotion is a separate human decision;
|
||||
do not trigger the promotion workflow yourself.
|
||||
- Update E2E journeys in the same PR when changing user-facing behaviour.
|
||||
- Do not attempt to start a local server to test the application; use unit checks
|
||||
and the configured integration environment.
|
||||
@@ -0,0 +1,107 @@
|
||||
# Architecture
|
||||
|
||||
This describes the implementation as reviewed on 2026-09-14. It distinguishes
|
||||
current behaviour from improvements still to be implemented.
|
||||
|
||||
## Request flow
|
||||
|
||||
```text
|
||||
Browser → Next.js public routes
|
||||
├─ /api/* proxy → FastAPI → cached DataFrames / PostgreSQL marts
|
||||
│ ├─ Typesense (search and suggestions)
|
||||
│ └─ postcodes.io (postcode lookup)
|
||||
└─ /admin, /cms-api, /blog → Payload → payload schema + media volume
|
||||
|
||||
Next.js server rendering → FastAPI directly through FASTAPI_URL
|
||||
```
|
||||
|
||||
`nextjs-app/lib/api.ts` contains typed fetch wrappers and revalidation defaults.
|
||||
The proxy is `nextjs-app/app/(frontend)/api/[...path]/route.ts`. Payload uses
|
||||
`/cms-api` so its routes do not collide with the FastAPI proxy. The proxy denies
|
||||
`/api/flags`; server-side rendering reads flags directly from FastAPI.
|
||||
|
||||
## Data ownership
|
||||
|
||||
| Layer | Owner and role |
|
||||
|---|---|
|
||||
| Source data | GIAS, DfE EES, Ofsted, finance, deprivation and council admission-distance sources |
|
||||
| `raw` | Singer taps and the PostgreSQL target configured in `pipeline/meltano.yml` |
|
||||
| Staging/intermediate/marts | dbt models in `pipeline/transform`; marts are materialized tables |
|
||||
| `marts.dim_school`, `marts.dim_location` | School identity and location, filtered to supported England establishments |
|
||||
| `marts.fact_*` | Performance and supplementary datasets; coverage and years vary |
|
||||
| Typesense `schools` alias | Search documents built by `pipeline/scripts/sync_typesense.py` |
|
||||
| `payload` | CMS collections and migrations in `nextjs-app/`; independent of dbt |
|
||||
| Media volume | Uploaded blog media; requires backup and cannot be regenerated from school datasets |
|
||||
|
||||
`backend/models.py` maps existing marts for reading. It does not create the school
|
||||
schema. There is no startup schema-version migration or CSV reimport. Payload's
|
||||
`nextjs-app/migrations/` is active and must not be confused with the removed
|
||||
legacy backend migration code.
|
||||
|
||||
Coordinates normally come from GIAS British National Grid coordinates transformed
|
||||
by PostGIS in `dim_location.sql`. `pipeline/scripts/geocode_postcodes.py` is a
|
||||
manual fallback utility, not a task wired into the current school-data DAG.
|
||||
Backend postcode searches also use postcodes.io; that lookup does not populate
|
||||
school coordinates in the database.
|
||||
|
||||
## Backend boundaries
|
||||
|
||||
- `app.py`: routes, middleware, search filtering, sitemap/place publication and response assembly.
|
||||
- `data_loader.py`: SQL loading, process-local DataFrame caches, Typesense calls,
|
||||
postcode lookups, supplementary queries and benchmark calculation.
|
||||
- `database.py`: synchronous SQLAlchemy engine and sessions.
|
||||
- `schemas.py`: metric definitions, column mappings and display metadata; despite
|
||||
its name this is not a collection of Pydantic API response models.
|
||||
- `places.py` and `localities.py`: place registry and curated locality information.
|
||||
- `flags.py`: Unleash-backed feature flags, disabled when no server is configured.
|
||||
- `gias_codes.py` / `ofsted_codes.py`: source-code translation and display rules.
|
||||
|
||||
Search starts from a cached latest-row-per-school snapshot. Detail pages read
|
||||
history from the full DataFrame and supplementary data from marts. Comparisons
|
||||
batch supplementary queries across selected URNs. Async routes still contain
|
||||
synchronous dependency calls; a fully asynchronous database layer is not present.
|
||||
|
||||
## Frontend boundaries
|
||||
|
||||
`app/(frontend)` owns the public root layout and pages. `app/(payload)` owns the
|
||||
CMS root layout. Do not add a shared `app/layout.tsx`: these groups deliberately
|
||||
have separate root layouts. Root metadata files remain in `app/`.
|
||||
|
||||
Server pages fetch initial data and pass it to client views. Client state uses
|
||||
React hooks, URL search parameters and the comparison context/localStorage.
|
||||
There is no SWR dependency. Leaflet maps are loaded through dynamic wrappers;
|
||||
Chart.js renders performance and comparison charts.
|
||||
|
||||
`components/school/` contains detail sections, with section decisions and data
|
||||
preparation in `lib/schoolSections.ts`. `lib/types.ts` contains manually maintained
|
||||
API types. `payload-types.ts` and the Payload import map are generated artifacts.
|
||||
|
||||
## Publication and caching today
|
||||
|
||||
1. Airflow DAGs extract and validate source data, then run selected dbt builds.
|
||||
2. Relevant DAGs rebuild Typesense and swap the `schools` alias.
|
||||
3. They call `POST /api/admin/reload` with `X-API-Key` to refresh school DataFrames.
|
||||
4. A separate weekly sitemap DAG calls `POST /api/admin/regenerate-sitemap`,
|
||||
rebuilding places and sitemaps.
|
||||
|
||||
GIAS is scheduled daily, Ofsted monthly, and annual datasets are manually
|
||||
triggered. The DAG definitions are authoritative for selectors and dependencies.
|
||||
|
||||
Caches exist in several independent layers: backend DataFrames and registries,
|
||||
backend HTTP Cache-Control/ETags, Next.js fetch/page revalidation, and browser or
|
||||
shared HTTP caches where configured. Place fetches request a one-week revalidation
|
||||
interval. HTTP ETags are computed after route execution, not before database work.
|
||||
|
||||
Known limitations: reload clears the old DataFrames before verifying replacement
|
||||
data; places/sitemaps refresh separately; Next.js caches are not explicitly purged
|
||||
by the pipeline; Typesense import results are not validated before alias publication.
|
||||
Do not describe this sequence as an atomic dataset release. These are follow-up
|
||||
reliability tasks, not changes implemented by the documentation cleanup.
|
||||
|
||||
## Deployment references
|
||||
|
||||
See [DEPLOY.md](DEPLOY.md). PR checks include frontend typechecking/tests, backend
|
||||
unit tests, image builds and AI review. Staging journeys run after merging.
|
||||
Production promotion retags a selected commit's images. Current health polling
|
||||
checks HTTP success, not the deployed commit identity; overlapping staging runs
|
||||
remain a release-verification concern.
|
||||
@@ -0,0 +1,94 @@
|
||||
# Development and validation
|
||||
|
||||
## Prerequisites and environment boundaries
|
||||
|
||||
Use a feature branch. The deployed stack is the integration environment; do not
|
||||
assume a local server can run from a fresh checkout. This cleanup did not start
|
||||
local servers or provision databases. Unit tests use fixtures and mocks.
|
||||
|
||||
The current versions are not yet aligned:
|
||||
|
||||
| Component | Container | PR checks |
|
||||
|---|---|---|
|
||||
| Backend | Python 3.11 | Python 3.12 |
|
||||
| Frontend | Node 24 | Node 22 |
|
||||
| Pipeline | Python 3.13 | Pipeline image build |
|
||||
|
||||
Use the component's container version when reproducing deployment behaviour.
|
||||
The backend dependency pins predate Python 3.14; do not assume the system Python
|
||||
can install or run them. Version alignment is a separate maintenance task.
|
||||
|
||||
## Frontend checks
|
||||
|
||||
```sh
|
||||
cd nextjs-app
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test -- --runInBand
|
||||
```
|
||||
|
||||
`npm run build` is the production build check. There is no `lint` script.
|
||||
Tests live in `__tests__/` and use Jest/React Testing Library. These checks do not
|
||||
prove that live PostgreSQL queries, Typesense or a deployed proxy work.
|
||||
|
||||
The frontend `.env.example` documents runtime variables. Browser traffic normally
|
||||
uses `/api`; `FASTAPI_URL` is an absolute server-side URL ending in `/api`.
|
||||
Payload additionally needs `DATABASE_URL` and `PAYLOAD_SECRET` when used at runtime.
|
||||
Never commit credentials or real `.env` files.
|
||||
|
||||
## Backend checks
|
||||
|
||||
From the repository root, using an available Python 3.11 or 3.12 interpreter:
|
||||
|
||||
```sh
|
||||
python3.11 -m venv /tmp/schoolcompare-backend-venv
|
||||
/tmp/schoolcompare-backend-venv/bin/python -m pip install -r requirements.txt pytest 'httpx<0.28'
|
||||
/tmp/schoolcompare-backend-venv/bin/python -m pytest backend/tests -q
|
||||
```
|
||||
|
||||
Substitute `python3.12` if matching PR CI. The test dependencies above match the
|
||||
current workflow; they are not yet captured in a dedicated development lockfile.
|
||||
Backend configuration is defined in `backend/config.py`; `.env.example` documents
|
||||
commonly used values. `ALLOWED_ORIGINS` uses a JSON array, not a comma-separated string.
|
||||
|
||||
## Data and pipeline work
|
||||
|
||||
The app needs populated `marts.*` tables. A new Postgres instance alone is not a
|
||||
working school-data environment. Use the existing managed pipeline or an approved
|
||||
snapshot; the removed CSV importer cannot build the current schema.
|
||||
|
||||
The pipeline container includes Meltano, dbt/Postgres, Airflow and the custom taps.
|
||||
Airflow commands/selectors live in `pipeline/dags/`. Schema tests live in
|
||||
`pipeline/transform/tests/` and model YAML files. Run the relevant `dbt build`
|
||||
selector in an isolated data environment for model changes; it writes tables and
|
||||
is not a read-only smoke test. Prefer `python -m dbt.cli.main` as the DAGs do.
|
||||
|
||||
GIAS dictionaries are generated together by
|
||||
`pipeline/scripts/generate_gias_codes.py`. The backend and pipeline copies are
|
||||
intentional; `backend/tests/test_gias_codes.py` checks that they stay identical.
|
||||
|
||||
For Payload collection/editor changes, run `npm run generate:importmap` in
|
||||
`nextjs-app/` and include the generated map. Preserve CMS migrations and the
|
||||
separate `payload` schema. See [publishing](../nextjs-app/docs/PUBLISHING.md).
|
||||
|
||||
## End-to-end checks
|
||||
|
||||
Against an existing, authorised test environment:
|
||||
|
||||
```sh
|
||||
cd e2e
|
||||
npm ci
|
||||
npx playwright install chromium
|
||||
BASE_URL=https://your-test-environment.example npx playwright test
|
||||
```
|
||||
|
||||
The suite does not start a web server. CI installs Chromium with system dependencies
|
||||
and runs against staging. Use the configured staging target: `docs/DEPLOY.md`
|
||||
records the public staging proxy limitation. User-visible behaviour changes should
|
||||
update the corresponding journeys.
|
||||
|
||||
## Before requesting review
|
||||
|
||||
Run checks relevant to the change, inspect `git diff --check`, and report checks
|
||||
that could not run. Do not publish or promote as part of local validation.
|
||||
[DEPLOY.md](DEPLOY.md) documents the PR and human promotion gates.
|
||||
@@ -0,0 +1,89 @@
|
||||
# Legacy and unused-code inventory
|
||||
|
||||
Reviewed 2026-09-14. This inventory records source evidence, not production usage
|
||||
telemetry. A command with no repository caller may still be run manually or from
|
||||
an external scheduler. Historical specs and prototypes are not runtime imports.
|
||||
|
||||
## Method and scope
|
||||
|
||||
Searched backend imports, tests, CLI scripts, Airflow DAGs, Meltano configuration,
|
||||
Gitea workflows, Dockerfiles and documentation. For frontend candidates, inspected
|
||||
TypeScript imports, re-exports, literal dynamic imports and `require` calls,
|
||||
resolving relative and `@/` paths while excluding tests, dependencies and build
|
||||
output. Checked candidates again with text searches including tests.
|
||||
|
||||
Next.js route files, generated Payload import-map entries and plugin discovery
|
||||
are entry points even without ordinary imports. This is why a zero-import count
|
||||
alone is not sufficient grounds for deletion. Computed imports and external
|
||||
operators are outside this static audit.
|
||||
|
||||
## Removed in this cleanup
|
||||
|
||||
These names are recorded for Git-history lookup; they are no longer file links.
|
||||
|
||||
| Removed path or symbol | Evidence and replacement |
|
||||
|---|---|
|
||||
| `backend/migration.py` | Imported `School` and `SchoolResult`, which no longer exist in `backend/models.py`. Only the legacy CSV CLI imported it. Current tables are built by dbt. |
|
||||
| `backend/version.py` | Only the legacy importer consumed `SCHEMA_VERSION`. FastAPI lifespan does not perform version-triggered imports. This is unrelated to active Payload migrations. |
|
||||
| `scripts/migrate_csv_to_db.py` | Imported removed `init_db`/`set_db_schema_version` helpers and the obsolete models indirectly. No runtime, DAG or workflow calls it. Use the managed pipeline for current marts. |
|
||||
| `scripts/geocode_schools.py` | Imported the removed `School` ORM model. No pipeline/workflow calls it. Coordinates now come from GIAS/PostGIS; a separate mart-aware manual utility remains under `pipeline/scripts/`. |
|
||||
| `backend.data_loader.haversine_distance` | No callers. Search uses its inline vectorised NumPy calculation. |
|
||||
| `nextjs-app/lib/api.ts: fetcher` | No callers; SWR is not installed. Application fetches use the named API wrappers. |
|
||||
| `nextjs-app/lib/api.ts: kmToMiles` | No callers. `calculateDistance` remains because `CutoffMapPanel` uses it. |
|
||||
|
||||
The removed command files could not import successfully against the current
|
||||
backend. This cleanup does not run replacements, migrate data or modify databases.
|
||||
Their previous implementations remain recoverable from Git history.
|
||||
|
||||
## Unused candidates retained for a separate cleanup
|
||||
|
||||
| Candidate | Evidence | Recommended next step |
|
||||
|---|---|---|
|
||||
| `nextjs-app/components/LoadingSkeleton.tsx` and its CSS | No application or test imports found. | Remove together after confirming no planned use. |
|
||||
| `nextjs-app/components/Pagination.tsx` and its CSS | No application or test imports found; HomeView implements load-more behaviour. | Remove as a pair if numbered pagination will not return. |
|
||||
| `nextjs-app/components/SchoolCard.tsx` and its CSS | Imported by its own tests, not application code. HomeView uses SchoolRow/SecondarySchoolRow. | Decide whether to retire the card design; if removed, remove its dedicated tests as well. Passing tests do not establish runtime use. |
|
||||
| `backend/database.py: get_db`, `get_db_session` | No remaining callers after removing the importer. Current code creates SessionLocal directly. | Either adopt these helpers during session-lifecycle cleanup or remove them; do not rewrite active sessions in a documentation change. |
|
||||
| `backend/schemas.py: COLUMN_MAPPINGS`, `NULL_VALUES`, `LA_CODE_TO_NAME` | No remaining Python consumers found after importer removal. Other constants in this module are active. | Remove individual constants after checking external data utilities; retain the module. |
|
||||
| `backend/config.py: data_dir`, `max_page_size`, `rate_limit_burst` | No active consumers found. `default_page_size` appears only in a branch that expects None, although the route supplies a concrete default. | Reconcile settings with route validation in a focused API change. |
|
||||
|
||||
## Legacy/manual paths requiring operational verification
|
||||
|
||||
| Path | Status and reason to retain for now |
|
||||
|---|---|
|
||||
| FastAPI `/`, `/compare`, `/rankings`, `/favicon.svg`, `/robots.txt`, and conditional `/static` | Old frontend-serving routes reference a `frontend/` directory absent from the checkout and backend image. Next.js owns these public surfaces. Removal changes externally callable routes, so first check proxy/operator usage and define replacement responses. |
|
||||
| `scripts/fetch_real_data.py`, `scripts/download_data.py` | Historical standalone CSV utilities. The fetch script targets Wandsworth/Merton; neither is wired into the managed pipeline. Marked historical, retained pending confirmation of manual use. |
|
||||
| `pipeline/scripts/geocode_postcodes.py` | Mart-aware postcode fallback, not called by the current DAGs. Do not confuse it with the removed legacy ORM geocoder. Verify the target schema before manual use. |
|
||||
| `docker-compose.yml` | Uses unpublished `:latest` release tags and lacks frontend Payload DB/secret/media configuration. Retained as an old development topology, not recommended onboarding. |
|
||||
| `nextjs-app/docker-compose.yml` | Standalone legacy recipe with old backend port assumptions and no CMS persistence setup. Retained until its consumers are checked. |
|
||||
| `MIGRATION_SUMMARY.md`, `docs/superpowers/`, `mockups/` | Historical designs and prototypes. Retain as history; do not follow as current deployment instructions. |
|
||||
| `scripts/sql/drop_fact_parent_view.sql` | One-off maintenance SQL. Not an application entry point; repository call-site searches cannot establish whether it is still needed operationally. |
|
||||
|
||||
## Active code that can look obsolete
|
||||
|
||||
- `backend/data_loader.py` older-mart query fallbacks are covered by backend tests
|
||||
and support databases at different migration stages. Remove only after verifying
|
||||
the deployed schemas in every supported environment.
|
||||
- `backend/gias_codes.py` and `pipeline/scripts/gias_codes.py` are intentionally
|
||||
generated copies for separate runtime images. Their parity is tested.
|
||||
- `nextjs-app/migrations/`, `payload-types.ts` and the Payload import map are active
|
||||
CMS artifacts, not remnants of the removed school importer.
|
||||
- `get_available_years`, `get_available_local_authorities` and `get_schools_count`
|
||||
in `data_loader.py` are called through `get_data_info`, which serves the backend
|
||||
data-info endpoint. They are not dead functions.
|
||||
- `get_supplementary_data` is an intentional single-school wrapper around the
|
||||
batch implementation.
|
||||
- `pipeline/transform` models named `legacy` can be active data sources: annual
|
||||
DAG selectors explicitly include legacy KS2/KS4 lineage. Names alone do not
|
||||
establish obsolescence.
|
||||
|
||||
## Suggested next passes
|
||||
|
||||
1. Decide the fate of the three unused UI components and remove paired assets/tests.
|
||||
2. Consolidate backend session usage and remove abandoned settings/constants.
|
||||
3. Verify external consumers, then retire static-serving API routes and old compose recipes.
|
||||
4. Audit manual data utilities with pipeline operators before deleting them.
|
||||
5. Revisit compatibility fallbacks only after documenting supported schema versions.
|
||||
|
||||
Validation for this cleanup should include frontend typechecking/tests, Python
|
||||
syntax checks, reference searches and documentation link checks. Live database,
|
||||
external scheduler and deployed route usage require separate integration evidence.
|
||||
+113
-1
@@ -1935,6 +1935,63 @@ async function firstPlaceOfKind(page: Page, kind: string) {
|
||||
return hit as { kind: string; slug: string; name: string; count: number };
|
||||
}
|
||||
|
||||
/**
|
||||
* The round trip. Place pages always linked down to school pages; school
|
||||
* pages linked nowhere on the site, so the ~27k of them that carry most of
|
||||
* the inbound authority stranded it — their only anchor pointed at the
|
||||
* school's own website.
|
||||
*
|
||||
* Asserting both directions is the point. A one-way link is what already
|
||||
* existed and is not what this journey is for.
|
||||
*/
|
||||
test('a school page links back into the location layer, and the place page links down', async ({ page }) => {
|
||||
const town = await firstPlaceOfKind(page, 'town');
|
||||
|
||||
// Start from the place page and take its first school, so the pair is
|
||||
// guaranteed to be genuinely related rather than a hardcoded guess.
|
||||
await page.goto(`/schools/${town.slug}`);
|
||||
const schoolHref = await page.locator('a[href^="/school/"]').first()
|
||||
.getAttribute('href');
|
||||
expect(schoolHref, 'the town page listed no school to follow').toBeTruthy();
|
||||
|
||||
await page.goto(schoolHref!);
|
||||
|
||||
// Down: the school page must offer a link back to the town it sits in.
|
||||
const backToTown = page.locator(`a[href="/schools/${town.slug}"]`);
|
||||
await expect(backToTown).toHaveCount(1);
|
||||
await expect(backToTown).toBeVisible();
|
||||
|
||||
// The anchor says what it leads to, which is worth more than "see more".
|
||||
await expect(backToTown).toContainText(town.name, { ignoreCase: true });
|
||||
await expect(backToTown).toContainText(/\d+ schools?/);
|
||||
|
||||
// And the breadcrumb resolves the school into a real hierarchy.
|
||||
const blocks = await page.locator('script[type="application/ld+json"]')
|
||||
.allTextContents();
|
||||
const graph = blocks.join(' ');
|
||||
expect(graph).toContain('"BreadcrumbList"');
|
||||
// The narrower type, not the EducationalOrganization parent it used to be.
|
||||
expect(graph).toContain('"School"');
|
||||
|
||||
/*
|
||||
* The phase variants are the pages this most needs to reach: ~950 of them
|
||||
* were once reachable by nothing at all, absent from every sitemap and
|
||||
* unlinked from the place page. Conditional because not every school sits
|
||||
* in a town that publishes one.
|
||||
*/
|
||||
const phaseLink = page.locator(`a[href^="/schools/${town.slug}/"]`).first();
|
||||
if (await phaseLink.count()) {
|
||||
const phaseHref = await phaseLink.getAttribute('href');
|
||||
expect((await page.request.get(phaseHref!)).status()).toBe(200);
|
||||
await expect(phaseLink).toContainText(/primary|secondary/);
|
||||
}
|
||||
|
||||
// Following it lands on a real page, not a 404.
|
||||
await backToTown.click();
|
||||
await page.waitForURL(new RegExp(`/schools/${town.slug}$`));
|
||||
await expect(page.locator('h1')).toContainText(town.name, { ignoreCase: true });
|
||||
});
|
||||
|
||||
for (const [kind, prefix, article] of [
|
||||
['town', '/schools/', 'a'],
|
||||
['authority', '/schools/authority/', 'an'],
|
||||
@@ -2560,8 +2617,56 @@ test('the destinations section never claims a pupil stayed at this school', asyn
|
||||
* These journeys assert the load-bearing parts of that — a name, a face, the
|
||||
* honesty claim, and a resolvable Person entity — rather than exact copy,
|
||||
* which will be edited.
|
||||
*
|
||||
* Both are behind flags (about_page, blog), so each has a lit journey and a
|
||||
* dark one. Flag state is read from the observable effect rather than from
|
||||
* /api/flags, which the public proxy denies on purpose — the same approach
|
||||
* distanceFeatureIsOn() takes above.
|
||||
*/
|
||||
async function aboutPageIsOn(page: Page): Promise<boolean> {
|
||||
return (await page.request.get('/about')).ok();
|
||||
}
|
||||
|
||||
async function blogIsOn(page: Page): Promise<boolean> {
|
||||
return (await page.request.get('/blog')).ok();
|
||||
}
|
||||
|
||||
test('with the about page off, it is absent rather than empty', async ({ page }) => {
|
||||
test.skip(await aboutPageIsOn(page), 'the about_page flag is on in this environment');
|
||||
|
||||
// Dark means the URL does not exist, not that it renders empty: a 404 is
|
||||
// what stops a crawler keeping the page in its index.
|
||||
expect((await page.request.get('/about')).status()).toBe(404);
|
||||
|
||||
// A footer link into a 404 is the failure this flag has to avoid.
|
||||
await page.goto('/');
|
||||
await expect(page.locator('footer a[href="/about"]')).toHaveCount(0);
|
||||
|
||||
// And a sitemap must never advertise a URL that 404s.
|
||||
const sitemap = await page.request.get('/content-sitemap.xml');
|
||||
expect(await sitemap.text()).not.toContain('/about');
|
||||
});
|
||||
|
||||
test('with the blog off, it is absent rather than empty', async ({ page }) => {
|
||||
test.skip(await blogIsOn(page), 'the blog flag is on in this environment');
|
||||
|
||||
expect((await page.request.get('/blog')).status()).toBe(404);
|
||||
expect((await page.request.get('/blog/rss.xml')).status()).toBe(404);
|
||||
|
||||
await page.goto('/');
|
||||
await expect(page.locator('footer a[href="/blog"]')).toHaveCount(0);
|
||||
|
||||
const sitemap = await page.request.get('/content-sitemap.xml');
|
||||
expect(await sitemap.text()).not.toContain('/blog');
|
||||
|
||||
// The admin panel is deliberately NOT flagged: posts have to be writable
|
||||
// before the blog is readable, or there is nothing to turn on.
|
||||
expect((await page.request.get('/admin')).status()).not.toBe(404);
|
||||
});
|
||||
|
||||
test('the about page names a human author and is reachable from the footer', async ({ page }) => {
|
||||
test.skip(!(await aboutPageIsOn(page)), 'the about_page flag is off in this environment');
|
||||
|
||||
await page.goto('/');
|
||||
const aboutLink = page.locator('footer a[href="/about"]');
|
||||
await expect(aboutLink).toBeVisible();
|
||||
@@ -2585,6 +2690,8 @@ test('the about page names a human author and is reachable from the footer', asy
|
||||
});
|
||||
|
||||
test('the blog lists posts and each one renders with a byline', async ({ page }) => {
|
||||
test.skip(!(await blogIsOn(page)), 'the blog flag is off in this environment');
|
||||
|
||||
await page.goto('/blog');
|
||||
await expect(page.getByRole('heading', { level: 1 })).toBeVisible();
|
||||
|
||||
@@ -2612,8 +2719,13 @@ test('the admin panel is not indexable', async ({ page }) => {
|
||||
|
||||
test('the content sitemap lists the about page and is advertised in robots', async ({ page }) => {
|
||||
const sitemap = await page.request.get('/content-sitemap.xml');
|
||||
// Served whatever the flags say: robots.txt names it unconditionally, and
|
||||
// with both dark it is a valid empty urlset rather than a 404.
|
||||
expect(sitemap.ok()).toBeTruthy();
|
||||
expect(await sitemap.text()).toContain('/about');
|
||||
|
||||
if (await aboutPageIsOn(page)) {
|
||||
expect(await sitemap.text()).toContain('/about');
|
||||
}
|
||||
|
||||
// The school corpus sitemap is proxied from FastAPI; this one is Next's.
|
||||
// robots.txt must advertise both or the blog never gets discovered.
|
||||
|
||||
+12
-4
@@ -1,8 +1,16 @@
|
||||
# API Configuration
|
||||
NEXT_PUBLIC_API_URL=http://localhost:8000/api
|
||||
# Browser requests use the same-origin Next.js proxy.
|
||||
NEXT_PUBLIC_API_URL=/api
|
||||
|
||||
# Production API URL (for deployment)
|
||||
# NEXT_PUBLIC_API_URL=https://api.schoolcompare.co.uk/api
|
||||
# Absolute URL for server-side fetching and the proxy; include /api.
|
||||
# In the managed container network this is http://backend:80/api (staging differs).
|
||||
FASTAPI_URL=http://localhost:8000/api
|
||||
|
||||
# Payload CMS runtime configuration. Use the managed environment's database;
|
||||
# Payload owns the payload schema, independently of the school marts.
|
||||
DATABASE_URL=postgresql://schoolcompare:CHANGE_THIS_PASSWORD@localhost:5432/schoolcompare
|
||||
# Generate a secret: python -c "import secrets; print(secrets.token_urlsafe(32))"
|
||||
# Use distinct secrets for staging and production.
|
||||
PAYLOAD_SECRET=CHANGE_THIS_TO_A_SECURE_RANDOM_SECRET
|
||||
|
||||
# Node Environment
|
||||
NODE_ENV=development
|
||||
+12
-288
@@ -1,291 +1,15 @@
|
||||
# Deployment Guide
|
||||
# Frontend deployment
|
||||
|
||||
This guide covers deployment options for the SchoolCompare Next.js application.
|
||||
Next.js and Payload run in the same frontend container. The maintained deployment
|
||||
procedure is [docs/DEPLOY.md](../docs/DEPLOY.md), with the production and staging
|
||||
Portainer compose files at the repository root.
|
||||
|
||||
## Deployment Options
|
||||
The frontend Dockerfile builds a standalone Next.js image. Runtime configuration
|
||||
supplies `FASTAPI_URL`, `DATABASE_URL` and `PAYLOAD_SECRET`; uploaded CMS media is
|
||||
persisted in a volume. Promote the built image through the repository's Gitea
|
||||
workflow after human staging approval.
|
||||
|
||||
### Option 1: Vercel (Recommended for Next.js)
|
||||
|
||||
Vercel is the easiest and most optimized platform for Next.js applications.
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. **Install Vercel CLI**:
|
||||
```bash
|
||||
npm install -g vercel
|
||||
```
|
||||
|
||||
2. **Login to Vercel**:
|
||||
```bash
|
||||
vercel login
|
||||
```
|
||||
|
||||
3. **Deploy**:
|
||||
```bash
|
||||
vercel --prod
|
||||
```
|
||||
|
||||
4. **Configure Environment Variables** in Vercel dashboard:
|
||||
- `NEXT_PUBLIC_API_URL`: Your FastAPI endpoint (e.g., `https://api.schoolcompare.co.uk/api`)
|
||||
- `FASTAPI_URL`: Same as above for server-side requests
|
||||
|
||||
#### Benefits:
|
||||
- Automatic HTTPS
|
||||
- Global CDN
|
||||
- Zero-config deployment
|
||||
- Automatic preview deployments
|
||||
- Built-in analytics
|
||||
|
||||
---
|
||||
|
||||
### Option 2: Docker (Self-hosted)
|
||||
|
||||
Deploy using Docker containers for full control.
|
||||
|
||||
#### Prerequisites:
|
||||
- Docker 20+
|
||||
- Docker Compose 2+
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. **Build Docker Image**:
|
||||
```bash
|
||||
docker build -t schoolcompare-nextjs:latest .
|
||||
```
|
||||
|
||||
2. **Run with Docker Compose**:
|
||||
```bash
|
||||
# Create .env file with production variables
|
||||
echo "NEXT_PUBLIC_API_URL=https://api.schoolcompare.co.uk/api" > .env
|
||||
echo "FASTAPI_URL=http://backend:8000/api" >> .env
|
||||
|
||||
# Start services
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
3. **Verify Deployment**:
|
||||
```bash
|
||||
curl http://localhost:3000
|
||||
```
|
||||
|
||||
#### Environment Variables:
|
||||
- `NEXT_PUBLIC_API_URL`: Public API endpoint (client-side)
|
||||
- `FASTAPI_URL`: Internal API endpoint (server-side)
|
||||
- `NODE_ENV`: `production`
|
||||
|
||||
---
|
||||
|
||||
### Option 3: PM2 (Node.js Process Manager)
|
||||
|
||||
Deploy directly on a Node.js server using PM2.
|
||||
|
||||
#### Prerequisites:
|
||||
- Node.js 24+
|
||||
- PM2 (`npm install -g pm2`)
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. **Build Application**:
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
2. **Create PM2 Ecosystem File** (`ecosystem.config.js`):
|
||||
```javascript
|
||||
module.exports = {
|
||||
apps: [{
|
||||
name: 'schoolcompare-nextjs',
|
||||
script: 'npm',
|
||||
args: 'start',
|
||||
cwd: '/path/to/nextjs-app',
|
||||
instances: 'max',
|
||||
exec_mode: 'cluster',
|
||||
env: {
|
||||
NODE_ENV: 'production',
|
||||
PORT: 3000,
|
||||
NEXT_PUBLIC_API_URL: 'https://api.schoolcompare.co.uk/api',
|
||||
FASTAPI_URL: 'http://localhost:8000/api',
|
||||
},
|
||||
}],
|
||||
};
|
||||
```
|
||||
|
||||
3. **Start with PM2**:
|
||||
```bash
|
||||
pm2 start ecosystem.config.js
|
||||
pm2 save
|
||||
pm2 startup
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Option 4: Nginx Reverse Proxy
|
||||
|
||||
Use Nginx as a reverse proxy in front of Next.js.
|
||||
|
||||
#### Nginx Configuration:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name schoolcompare.co.uk;
|
||||
|
||||
# Redirect to HTTPS
|
||||
return 301 https://$server_name$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name schoolcompare.co.uk;
|
||||
|
||||
# SSL Configuration
|
||||
ssl_certificate /etc/ssl/certs/schoolcompare.crt;
|
||||
ssl_certificate_key /etc/ssl/private/schoolcompare.key;
|
||||
|
||||
# Security Headers
|
||||
# frame-ancestors replaces X-Frame-Options so the analytics subdomain
|
||||
# (Umami heatmap/recorder) can embed the site in an iframe.
|
||||
add_header Content-Security-Policy "frame-ancestors 'self' https://analytics.schoolcompare.co.uk" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
|
||||
# Proxy to Next.js
|
||||
location / {
|
||||
proxy_pass http://localhost:3000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Proxy to FastAPI
|
||||
location /api/ {
|
||||
proxy_pass http://localhost:8000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Cache static files
|
||||
location /_next/static/ {
|
||||
proxy_pass http://localhost:3000;
|
||||
add_header Cache-Control "public, max-age=31536000, immutable";
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Pre-Deployment Checklist
|
||||
|
||||
- [ ] Run `npm run build` successfully
|
||||
- [ ] Run `npm test` - all tests pass
|
||||
- [ ] Environment variables configured
|
||||
- [ ] FastAPI backend accessible
|
||||
- [ ] Database migrations applied
|
||||
- [ ] SSL certificates configured (production)
|
||||
- [ ] Domain DNS configured
|
||||
- [ ] Monitoring/logging set up
|
||||
- [ ] Backup strategy in place
|
||||
|
||||
---
|
||||
|
||||
## Post-Deployment Verification
|
||||
|
||||
1. **Health Check**:
|
||||
```bash
|
||||
curl https://schoolcompare.co.uk
|
||||
```
|
||||
|
||||
2. **Test Routes**:
|
||||
- Home: `https://schoolcompare.co.uk/`
|
||||
- School Page: `https://schoolcompare.co.uk/school/100001`
|
||||
- Compare: `https://schoolcompare.co.uk/compare`
|
||||
- Rankings: `https://schoolcompare.co.uk/rankings`
|
||||
|
||||
3. **Check SEO**:
|
||||
- Sitemap: `https://schoolcompare.co.uk/sitemap.xml`
|
||||
- Robots: `https://schoolcompare.co.uk/robots.txt`
|
||||
|
||||
4. **Performance Audit**:
|
||||
- Run Lighthouse in Chrome DevTools
|
||||
- Target scores: 90+ for Performance, Accessibility, Best Practices, SEO
|
||||
|
||||
---
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Recommended Tools:
|
||||
- **Vercel Analytics** (if using Vercel)
|
||||
- **Sentry** for error tracking
|
||||
- **Google Analytics** for user analytics
|
||||
- **Uptime Robot** for uptime monitoring
|
||||
|
||||
### Health Check Endpoint:
|
||||
The application automatically serves health data at the root route.
|
||||
|
||||
---
|
||||
|
||||
## Rollback Procedure
|
||||
|
||||
### Vercel:
|
||||
```bash
|
||||
vercel rollback
|
||||
```
|
||||
|
||||
### Docker:
|
||||
```bash
|
||||
docker-compose down
|
||||
docker-compose up -d --force-recreate
|
||||
```
|
||||
|
||||
### PM2:
|
||||
```bash
|
||||
pm2 stop schoolcompare-nextjs
|
||||
# Restore previous build
|
||||
pm2 start schoolcompare-nextjs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Issue: API requests failing
|
||||
- **Solution**: Check `NEXT_PUBLIC_API_URL` and `FASTAPI_URL` environment variables
|
||||
- **Verify**: FastAPI backend is accessible from Next.js container/server
|
||||
|
||||
### Issue: Build fails
|
||||
- **Solution**: Check Node.js version (requires 24+)
|
||||
- **Clear cache**: `rm -rf .next node_modules && npm install && npm run build`
|
||||
|
||||
### Issue: Slow page loads
|
||||
- **Solution**: Enable caching in API calls
|
||||
- **Check**: Network latency to FastAPI backend
|
||||
- **Verify**: CDN is serving static assets
|
||||
|
||||
---
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- ✅ HTTPS enabled
|
||||
- ✅ Security headers configured (X-Frame-Options, CSP, etc.)
|
||||
- ✅ API keys in environment variables (never in code)
|
||||
- ✅ CORS properly configured
|
||||
- ✅ Rate limiting on API endpoints
|
||||
- ✅ Regular security updates
|
||||
- ✅ Dependency vulnerability scanning
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
For deployment issues, contact the DevOps team or refer to:
|
||||
- [Next.js Deployment Docs](https://nextjs.org/docs/deployment)
|
||||
- [Vercel Documentation](https://vercel.com/docs)
|
||||
- [Docker Documentation](https://docs.docker.com/)
|
||||
Earlier Vercel and standalone deployment recipes have been retired from this file
|
||||
because they do not describe the current CMS, persistence and promotion setup.
|
||||
See [development](../docs/DEVELOPMENT.md) for checks and
|
||||
[publishing](docs/PUBLISHING.md) for CMS operations.
|
||||
+43
-141
@@ -1,156 +1,58 @@
|
||||
# SchoolCompare Next.js Application
|
||||
# SchoolCompare frontend and CMS
|
||||
|
||||
Modern Next.js application for comparing primary school KS2 performance across England.
|
||||
Next.js App Router with React, TypeScript, CSS Modules, Chart.js, Leaflet and
|
||||
Payload CMS. It serves school search, comparisons, rankings, school/place detail
|
||||
pages and editorial content across England.
|
||||
|
||||
## Features
|
||||
Start with the [repository overview](../README.md),
|
||||
[architecture](../docs/ARCHITECTURE.md) and [development checks](../docs/DEVELOPMENT.md).
|
||||
|
||||
- **Server-Side Rendering (SSR)**: Fast initial page loads with pre-rendered content
|
||||
- **Individual School Pages**: Dedicated pages for each school with full SEO optimization
|
||||
- **Side-by-Side Comparison**: Compare up to 5 schools simultaneously
|
||||
- **School Rankings**: Top-performing schools by various metrics
|
||||
- **Interactive Maps**: Leaflet integration for geographic visualization
|
||||
- **Performance Charts**: Chart.js visualizations for historical data
|
||||
- **Responsive Design**: Mobile-first approach with full responsive support
|
||||
- **SEO Optimized**: Dynamic sitemaps, meta tags, and structured data
|
||||
## Source map
|
||||
|
||||
## Tech Stack
|
||||
| Path | Purpose |
|
||||
|---|---|
|
||||
| `app/(frontend)/` | Public root layout, server pages and FastAPI proxy |
|
||||
| `app/(payload)/` | Payload root layout, `/admin` and `/cms-api` |
|
||||
| `app/robots.ts`, `app/opengraph-image.tsx`, root icons | Site-wide metadata endpoints |
|
||||
| `components/` | Client views and reusable display components |
|
||||
| `components/school/` | School detail sections |
|
||||
| `lib/api.ts`, `lib/types.ts` | Fetch wrappers and manual school API types |
|
||||
| `lib/schoolSections.ts`, `lib/compareLogic.ts` | Presentation decisions and data preparation |
|
||||
| `context/`, `hooks/` | Comparison state, suggestion state and responsive behaviour |
|
||||
| `collections/`, `blocks/`, `migrations/` | CMS schema and production migrations |
|
||||
| `__tests__/` | Jest and React Testing Library tests |
|
||||
|
||||
- **Framework**: Next.js 16 (App Router)
|
||||
- **Language**: TypeScript 5
|
||||
- **Styling**: CSS Modules + CSS Variables
|
||||
- **State Management**: React Context API + URL state
|
||||
- **Data Fetching**: SWR (client-side) + Next.js fetch (server-side)
|
||||
- **Charts**: Chart.js + react-chartjs-2
|
||||
- **Maps**: Leaflet + react-leaflet
|
||||
- **Testing**: Jest + React Testing Library
|
||||
- **Validation**: Zod
|
||||
Do not introduce a shared `app/layout.tsx`: public pages and Payload have separate
|
||||
root layouts. Keep root metadata files outside the route groups.
|
||||
|
||||
## Getting Started
|
||||
## Data and state
|
||||
|
||||
### Prerequisites
|
||||
Server pages fetch initial data directly from `FASTAPI_URL`. Browser fetches use
|
||||
`/api` by default, forwarded by `app/(frontend)/api/[...path]/route.ts`.
|
||||
`FASTAPI_URL` must include `/api`. See `.env.example` for CMS and API settings.
|
||||
|
||||
- Node.js 24+ (using nvm recommended)
|
||||
- FastAPI backend running on port 8000
|
||||
State uses React hooks/context, URL search parameters and localStorage for the
|
||||
comparison basket. SWR is not installed. Maps use dynamic Leaflet wrappers.
|
||||
Revalidation intervals are configured in fetch wrappers and pages; they vary by
|
||||
resource. Backend reloads do not automatically invalidate every Next.js cache.
|
||||
|
||||
### Installation
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
npm install
|
||||
|
||||
# Copy environment variables
|
||||
cp .env.example .env.local
|
||||
|
||||
# Update .env.local with your configuration
|
||||
```
|
||||
|
||||
### Development
|
||||
|
||||
```bash
|
||||
# Start development server
|
||||
npm run dev
|
||||
|
||||
# Open http://localhost:3000
|
||||
```
|
||||
|
||||
### Building
|
||||
|
||||
```bash
|
||||
# Build for production
|
||||
```sh
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test -- --runInBand
|
||||
npm run build
|
||||
|
||||
# Start production server
|
||||
npm start
|
||||
```
|
||||
|
||||
### Testing
|
||||
`test:watch` and `test:coverage` are also available. There is no `lint` script.
|
||||
A running application needs the backend/data environment described in the
|
||||
[development guide](../docs/DEVELOPMENT.md).
|
||||
|
||||
```bash
|
||||
# Run tests
|
||||
npm test
|
||||
After CMS field or editor changes, run `npm run generate:importmap`. Keep
|
||||
`payload-types.ts` generated from the CMS schema rather than editing it by hand.
|
||||
The build must work without a database connection; avoid module-scope CMS queries
|
||||
and DB-backed `generateStaticParams` functions.
|
||||
|
||||
# Run tests in watch mode
|
||||
npm run test:watch
|
||||
|
||||
# Run tests with coverage
|
||||
npm run test:coverage
|
||||
```
|
||||
|
||||
### Linting
|
||||
|
||||
```bash
|
||||
# Run ESLint
|
||||
npm run lint
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
nextjs-app/
|
||||
├── app/ # App Router pages
|
||||
│ ├── layout.tsx # Root layout
|
||||
│ ├── page.tsx # Home page
|
||||
│ ├── compare/ # Compare page
|
||||
│ ├── rankings/ # Rankings page
|
||||
│ ├── school/[urn]/ # Individual school pages
|
||||
│ ├── sitemap.ts # Dynamic sitemap
|
||||
│ └── robots.ts # Robots.txt
|
||||
├── components/ # React components
|
||||
│ ├── SchoolCard.tsx # School card component
|
||||
│ ├── FilterBar.tsx # Search/filter controls
|
||||
│ ├── ComparisonView.tsx # Comparison interface
|
||||
│ ├── RankingsView.tsx # Rankings table
|
||||
│ └── ...
|
||||
├── lib/ # Utility libraries
|
||||
│ ├── api.ts # API client
|
||||
│ ├── types.ts # TypeScript types
|
||||
│ └── utils.ts # Helper functions
|
||||
├── hooks/ # Custom React hooks
|
||||
├── context/ # React Context providers
|
||||
├── styles/ # Global styles
|
||||
├── public/ # Static assets
|
||||
└── __tests__/ # Test files
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `NEXT_PUBLIC_API_URL` | Public API endpoint (client-side) | `http://localhost:8000/api` |
|
||||
| `FASTAPI_URL` | Server-side API endpoint | `http://localhost:8000/api` |
|
||||
| `NODE_ENV` | Environment mode | `development` |
|
||||
|
||||
## Performance Optimizations
|
||||
|
||||
- **Server-Side Rendering**: Initial HTML rendered on server
|
||||
- **Static Generation**: Where possible, pages are pre-generated
|
||||
- **Image Optimization**: Next.js Image component with AVIF/WebP support
|
||||
- **Code Splitting**: Automatic route-based code splitting
|
||||
- **Dynamic Imports**: Heavy components loaded on demand
|
||||
- **API Caching**: Configurable revalidation for data fetching
|
||||
- **Bundle Optimization**: Tree shaking and minification
|
||||
- **Compression**: Gzip compression enabled
|
||||
|
||||
## SEO Features
|
||||
|
||||
- **Dynamic Meta Tags**: Generated per page with Next.js Metadata API
|
||||
- **Open Graph**: Social media optimization
|
||||
- **JSON-LD**: Structured data for search engines
|
||||
- **Sitemap**: Auto-generated from database
|
||||
- **Robots.txt**: Search engine crawling rules
|
||||
- **Canonical URLs**: Duplicate content prevention
|
||||
|
||||
## Browser Support
|
||||
|
||||
- Chrome (latest)
|
||||
- Firefox (latest)
|
||||
- Safari (latest)
|
||||
- Edge (latest)
|
||||
|
||||
## License
|
||||
|
||||
Proprietary - SchoolCompare
|
||||
|
||||
## Support
|
||||
|
||||
For issues and questions, please contact the development team.
|
||||
See [publishing](docs/PUBLISHING.md) for CMS operations and
|
||||
[deployment](../docs/DEPLOY.md) for staging and production promotion.
|
||||
@@ -27,14 +27,27 @@ describe('BlogPosting structured data', () => {
|
||||
it('names the same Person entity the about page declares', () => {
|
||||
// By @id, not by repeating the person: search engines must resolve every
|
||||
// post and the about page to one author entity, or the site has several.
|
||||
const ld = blogPostingJsonLd(post);
|
||||
const ld = blogPostingJsonLd(post, { namedAuthor: true });
|
||||
expect(ld['@type']).toBe('BlogPosting');
|
||||
expect(ld.author['@id']).toBe('https://www.schoolcompare.co.uk/about#tudor');
|
||||
expect(ld.publisher['@id']).toBe('https://www.schoolcompare.co.uk#organization');
|
||||
});
|
||||
|
||||
it('attributes to the organization when the about page is dark', () => {
|
||||
/*
|
||||
* The two flags are independent, so blog-on-about-off is a reachable
|
||||
* state. The Person entity lives at /about#tudor and that URL 404s while
|
||||
* the flag is dark, so claiming it would declare an author that resolves
|
||||
* to nothing — worse for the blog's credibility than having no named
|
||||
* author at all. Attribute to the publisher instead.
|
||||
*/
|
||||
const ld = blogPostingJsonLd(post, { namedAuthor: false });
|
||||
expect(ld.author['@id']).toBe('https://www.schoolcompare.co.uk#organization');
|
||||
expect(JSON.stringify(ld)).not.toContain('/about');
|
||||
});
|
||||
|
||||
it('carries a self-referencing canonical url and the publish date', () => {
|
||||
const ld = blogPostingJsonLd(post);
|
||||
const ld = blogPostingJsonLd(post, { namedAuthor: true });
|
||||
expect(ld.url).toBe(
|
||||
'https://www.schoolcompare.co.uk/blog/what-the-data-cannot-tell-you',
|
||||
);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { metadata as homeMetadata } from '@/app/(frontend)/page';
|
||||
import { metadata as rankingsMetadata } from '@/app/(frontend)/rankings/page';
|
||||
import { metadata as admissionsMetadata } from '@/app/(frontend)/admissions/page';
|
||||
import { generateMetadata as compareMetadata } from '@/app/(frontend)/compare/page';
|
||||
import { metadata as rootMetadata } from '@/app/(frontend)/layout';
|
||||
|
||||
describe('canonical URLs', () => {
|
||||
it('the homepage canonicalises to the bare root', () => {
|
||||
@@ -128,3 +129,41 @@ describe('C1 snippet copy', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The share card must be declared, not inherited.
|
||||
*
|
||||
* `app/opengraph-image.tsx` is a metadata file convention, and it does attach
|
||||
* to routes in the app root segment — `_not-found` gets an og:image from it.
|
||||
* It does NOT attach to the site's pages, which live in the `(frontend)`
|
||||
* route group whose own layout is a root layout. Staging served og:title,
|
||||
* og:description, og:url, og:site_name and og:type and no og:image at all,
|
||||
* so every link pasted into a chat rendered bare.
|
||||
*
|
||||
* The file stays at the app root, because /robots.txt and /icon.png depend on
|
||||
* it being there. The site's root layout points at the route it generates.
|
||||
*/
|
||||
describe('the share card', () => {
|
||||
it('declares an opengraph image on the site root layout', () => {
|
||||
// No og:image means every link pasted into a chat renders bare.
|
||||
const images = rootMetadata.openGraph?.images;
|
||||
expect(images).toBeTruthy();
|
||||
expect(JSON.stringify(images)).toContain('/opengraph-image');
|
||||
});
|
||||
|
||||
it('declares a twitter image too', () => {
|
||||
// twitter.card is summary_large_image. Claiming a large-image card and
|
||||
// supplying no image is worse than claiming a summary card.
|
||||
// Metadata['twitter'] is a union and `card` is not on every member, so
|
||||
// this reads the serialised shape rather than narrowing the type.
|
||||
const twitter = JSON.stringify(rootMetadata.twitter);
|
||||
expect(twitter).toContain('summary_large_image');
|
||||
expect(twitter).toContain('/opengraph-image');
|
||||
});
|
||||
|
||||
it('resolves the card to an absolute url via metadataBase', () => {
|
||||
// The e2e journey does `new URL(ogUrl)`, which throws on a relative path.
|
||||
expect(rootMetadata.metadataBase?.toString())
|
||||
.toBe('https://www.schoolcompare.co.uk/');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* The footer is the only navigational route to /about and /blog, so it is
|
||||
* where a dark flag would otherwise leave a link into a 404.
|
||||
*
|
||||
* Both props default to false. A caller that forgets to pass them hides the
|
||||
* links, which is the direction that cannot break a page — the same reasoning
|
||||
* as backend/flags.py's "every flag defaults to False".
|
||||
*/
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { Footer } from '@/components/Footer';
|
||||
|
||||
describe('footer feature links', () => {
|
||||
it('links to both when both flags are on', () => {
|
||||
render(<Footer aboutEnabled blogEnabled />);
|
||||
expect(screen.getByRole('link', { name: /who's behind this/i }))
|
||||
.toHaveAttribute('href', '/about');
|
||||
expect(screen.getByRole('link', { name: /^blog$/i }))
|
||||
.toHaveAttribute('href', '/blog');
|
||||
});
|
||||
|
||||
it('omits the about link when that flag is dark', () => {
|
||||
render(<Footer blogEnabled />);
|
||||
expect(screen.queryByRole('link', { name: /who's behind this/i })).toBeNull();
|
||||
expect(screen.getByRole('link', { name: /^blog$/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('omits the blog link when that flag is dark', () => {
|
||||
render(<Footer aboutEnabled />);
|
||||
expect(screen.queryByRole('link', { name: /^blog$/i })).toBeNull();
|
||||
expect(screen.getByRole('link', { name: /who's behind this/i }))
|
||||
.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('drops the whole section when both are dark, not an empty heading', () => {
|
||||
// Shipping dark means the footer renders as it did before the feature
|
||||
// existed, not as a section with its contents removed.
|
||||
render(<Footer />);
|
||||
expect(screen.queryByRole('heading', { name: /^about$/i })).toBeNull();
|
||||
expect(screen.queryByRole('link', { name: /who's behind this/i })).toBeNull();
|
||||
expect(screen.queryByRole('link', { name: /^blog$/i })).toBeNull();
|
||||
});
|
||||
|
||||
it('defaults to dark when a caller passes nothing', () => {
|
||||
render(<Footer />);
|
||||
expect(screen.queryByRole('link', { name: /who's behind this/i })).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* The module that ends the stranding: before it, a school page's only anchor
|
||||
* pointed at the school's own website, so ~27k pages sent authority off-site
|
||||
* and none of it reached the location layer.
|
||||
*/
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { NearbyPlaces } from '@/components/school/NearbyPlaces';
|
||||
|
||||
const essex = { kind: 'authority', slug: 'essex', name: 'Essex', count: 480, url: '/schools/authority/essex', phases: [] };
|
||||
const brentwood = { kind: 'town', slug: 'brentwood', name: 'Brentwood', count: 37, url: '/schools/brentwood', phases: [] };
|
||||
const cm15 = { kind: 'outcode', slug: 'cm15', name: 'CM15', count: 12, url: '/schools/near/cm15', phases: [] };
|
||||
|
||||
describe('NearbyPlaces', () => {
|
||||
it('links to every place the school belongs to', () => {
|
||||
render(<NearbyPlaces places={[essex, brentwood, cm15]} />);
|
||||
|
||||
expect(screen.getByRole('link', { name: /Brentwood/ }))
|
||||
.toHaveAttribute('href', '/schools/brentwood');
|
||||
expect(screen.getByRole('link', { name: /Essex/ }))
|
||||
.toHaveAttribute('href', '/schools/authority/essex');
|
||||
expect(screen.getByRole('link', { name: /CM15/ }))
|
||||
.toHaveAttribute('href', '/schools/near/cm15');
|
||||
});
|
||||
|
||||
it('says how many schools each link leads to', () => {
|
||||
// An anchor that states its destination's size is worth more to a reader
|
||||
// and to a crawler than "see more".
|
||||
render(<NearbyPlaces places={[brentwood]} />);
|
||||
expect(screen.getByRole('link', { name: /37 schools in Brentwood/ }))
|
||||
.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders nothing at all when the school has no published places', () => {
|
||||
// Not an empty heading. A school whose town and authority both fall below
|
||||
// the threshold has nowhere to point, and the page should look as it did
|
||||
// before the module existed.
|
||||
const { container } = render(<NearbyPlaces places={[]} />);
|
||||
expect(container).toBeEmptyDOMElement();
|
||||
});
|
||||
|
||||
it('puts the narrowest place first, which is the most useful link', () => {
|
||||
// The API orders widest-first for the breadcrumb; a reader on a school
|
||||
// page wants its town before its county.
|
||||
render(<NearbyPlaces places={[essex, brentwood, cm15]} />);
|
||||
const hrefs = screen.getAllByRole('link').map((a) => a.getAttribute('href'));
|
||||
expect(hrefs.indexOf('/schools/brentwood'))
|
||||
.toBeLessThan(hrefs.indexOf('/schools/authority/essex'));
|
||||
});
|
||||
|
||||
it('handles a singular count without saying "1 schools"', () => {
|
||||
render(<NearbyPlaces places={[{ ...brentwood, count: 1 }]} />);
|
||||
expect(screen.getByRole('link', { name: /1 school in Brentwood/ }))
|
||||
.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('links the phase page the school appears on', () => {
|
||||
// "primary schools in brentwood" is the query these pages exist for.
|
||||
render(<NearbyPlaces places={[{
|
||||
...brentwood,
|
||||
phases: [{ phase: 'primary', count: 22, url: '/schools/brentwood/primary' }],
|
||||
}]} />);
|
||||
|
||||
expect(screen.getByRole('link', { name: /22 primary schools in Brentwood/ }))
|
||||
.toHaveAttribute('href', '/schools/brentwood/primary');
|
||||
});
|
||||
|
||||
it('links both phase pages for an all-through school', () => {
|
||||
render(<NearbyPlaces places={[{
|
||||
...brentwood,
|
||||
phases: [
|
||||
{ phase: 'primary', count: 22, url: '/schools/brentwood/primary' },
|
||||
{ phase: 'secondary', count: 9, url: '/schools/brentwood/secondary' },
|
||||
],
|
||||
}]} />);
|
||||
|
||||
expect(screen.getByRole('link', { name: /22 primary schools/ })).toBeInTheDocument();
|
||||
expect(screen.getByRole('link', { name: /9 secondary schools/ })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('keeps a phase link next to the place it belongs to', () => {
|
||||
// Grouping matters: "22 primary schools in Brentwood" directly after
|
||||
// "37 schools in Brentwood" reads as one place, not two unrelated links.
|
||||
render(<NearbyPlaces places={[essex, {
|
||||
...brentwood,
|
||||
phases: [{ phase: 'primary', count: 22, url: '/schools/brentwood/primary' }],
|
||||
}]} />);
|
||||
|
||||
const hrefs = screen.getAllByRole('link').map((a) => a.getAttribute('href'));
|
||||
expect(hrefs.indexOf('/schools/brentwood/primary'))
|
||||
.toBe(hrefs.indexOf('/schools/brentwood') + 1);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { getFlags } from '@/lib/flags';
|
||||
import { getFlags, FLAGS_REVALIDATE } from '@/lib/flags';
|
||||
|
||||
// jsdom provides no global fetch, so there is nothing for jest.spyOn to attach
|
||||
// to — assign it and restore the original afterwards. This is the first test
|
||||
@@ -31,4 +31,28 @@ describe('getFlags', () => {
|
||||
mockFetch(async () => ({ ok: false, status: 503 }));
|
||||
await expect(getFlags()).resolves.toEqual({});
|
||||
});
|
||||
|
||||
/*
|
||||
* Reading a flag pins the calling route's ISR floor: Next uses the LOWEST
|
||||
* revalidate among a route's fetches for the whole route. That is why the
|
||||
* revalidate is an argument rather than the constant.
|
||||
*
|
||||
* Every SEO route here declares `revalidate = 604800`. A gate that read
|
||||
* flags at the 300s default would drop the whole school and place corpus
|
||||
* from a weekly cache to a 5-minute one, which is a large origin-load
|
||||
* regression to pay for a feature flag.
|
||||
*/
|
||||
it('reads at the 300s floor by default', async () => {
|
||||
mockFetch(async () => ({ ok: true, json: async () => ({}) }));
|
||||
await getFlags();
|
||||
expect((global.fetch as jest.Mock).mock.calls[0][1])
|
||||
.toEqual({ next: { revalidate: FLAGS_REVALIDATE } });
|
||||
});
|
||||
|
||||
it('lets a caller pass its own route floor instead', async () => {
|
||||
mockFetch(async () => ({ ok: true, json: async () => ({}) }));
|
||||
await getFlags(604800);
|
||||
expect((global.fetch as jest.Mock).mock.calls[0][1])
|
||||
.toEqual({ next: { revalidate: 604800 } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* School pages had no BreadcrumbList and no links into the location layer.
|
||||
* Both are fixed by the same data — the `places` array the API now returns —
|
||||
* so they are tested together.
|
||||
*/
|
||||
import { schoolBreadcrumbJsonLd } from '@/lib/jsonld';
|
||||
|
||||
const essex = { kind: 'authority', slug: 'essex', name: 'Essex', count: 480, url: '/schools/authority/essex', phases: [] };
|
||||
const brentwood = { kind: 'town', slug: 'brentwood', name: 'Brentwood', count: 37, url: '/schools/brentwood', phases: [] };
|
||||
const outcode = { kind: 'outcode', slug: 'cm15', name: 'CM15', count: 12, url: '/schools/near/cm15', phases: [] };
|
||||
|
||||
describe('school breadcrumbs', () => {
|
||||
it('reads home to authority to town to school', () => {
|
||||
const ld = schoolBreadcrumbJsonLd({
|
||||
name: 'Brentwood School', url: '/school/100000-brentwood-school',
|
||||
places: [essex, brentwood],
|
||||
});
|
||||
|
||||
expect(ld['@type']).toBe('BreadcrumbList');
|
||||
expect(ld.itemListElement.map((i) => i.name))
|
||||
.toEqual(['schoolcompare', 'Essex', 'Brentwood', 'Brentwood School']);
|
||||
expect(ld.itemListElement.map((i) => i.position)).toEqual([1, 2, 3, 4]);
|
||||
});
|
||||
|
||||
it('skips a level the school has no published place for', () => {
|
||||
// A school whose town falls below the publish threshold has no town page.
|
||||
// The trail closes over the gap rather than linking to a 404.
|
||||
const ld = schoolBreadcrumbJsonLd({
|
||||
name: 'Lone School', url: '/school/1-lone-school', places: [essex],
|
||||
});
|
||||
|
||||
expect(ld.itemListElement.map((i) => i.name))
|
||||
.toEqual(['schoolcompare', 'Essex', 'Lone School']);
|
||||
expect(ld.itemListElement.map((i) => i.position)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it('omits outcodes, which are not a place a breadcrumb reads through', () => {
|
||||
// CM15 is a useful link in the module but nonsense in a trail: nobody
|
||||
// navigates Essex → CM15 → school.
|
||||
const ld = schoolBreadcrumbJsonLd({
|
||||
name: 'Brentwood School', url: '/school/100000-brentwood-school',
|
||||
places: [essex, brentwood, outcode],
|
||||
});
|
||||
|
||||
expect(JSON.stringify(ld)).not.toContain('cm15');
|
||||
});
|
||||
|
||||
it('still produces a valid trail when the school has no places at all', () => {
|
||||
const ld = schoolBreadcrumbJsonLd({
|
||||
name: 'Orphan School', url: '/school/2-orphan-school', places: [],
|
||||
});
|
||||
|
||||
expect(ld.itemListElement.map((i) => i.name)).toEqual(['schoolcompare', 'Orphan School']);
|
||||
});
|
||||
|
||||
it('uses absolute urls, as every other entity on the site does', () => {
|
||||
const ld = schoolBreadcrumbJsonLd({
|
||||
name: 'Brentwood School', url: '/school/100000-brentwood-school',
|
||||
places: [essex, brentwood],
|
||||
});
|
||||
|
||||
for (const item of ld.itemListElement) {
|
||||
expect(item.item).toMatch(/^https:\/\/www\.schoolcompare\.co\.uk\//);
|
||||
}
|
||||
// The root is the homepage: there is no /schools index page to link to.
|
||||
expect(ld.itemListElement[0].item).toBe('https://www.schoolcompare.co.uk/');
|
||||
});
|
||||
});
|
||||
@@ -38,8 +38,7 @@ describe('payload mount points', () => {
|
||||
});
|
||||
|
||||
it('isolates CMS tables in their own postgres schema', () => {
|
||||
// Blog content must sit outside `public`, where the app tables, Airflow's
|
||||
// metadata and scripts/migrate_csv_to_db.py --drop all live.
|
||||
// Blog content must stay separate from school marts and Airflow metadata.
|
||||
expect(CONFIG).toMatch(/schemaName:\s*['"]payload['"]/);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { Metadata } from 'next';
|
||||
import Image from 'next/image';
|
||||
import { notFound } from 'next/navigation';
|
||||
import { absoluteUrl } from '@/lib/site';
|
||||
import { getFlags } from '@/lib/flags';
|
||||
import { personJsonLd, organizationJsonLd } from '@/lib/jsonld';
|
||||
import styles from './About.module.css';
|
||||
|
||||
@@ -11,7 +13,18 @@ export const metadata: Metadata = {
|
||||
alternates: { canonical: absoluteUrl('/about') },
|
||||
};
|
||||
|
||||
export default function AboutPage() {
|
||||
/*
|
||||
* Gated on about_page. The default 300s read is the right floor here: this
|
||||
* page declares no revalidate of its own, so nothing is lost by it, and a flip
|
||||
* lands within five minutes.
|
||||
*
|
||||
* notFound(), not a redirect: while the flag is dark this URL does not exist,
|
||||
* and a 404 is what tells a crawler not to keep it.
|
||||
*/
|
||||
export default async function AboutPage() {
|
||||
const flags = await getFlags();
|
||||
if (flags.about_page !== true) notFound();
|
||||
|
||||
const jsonLd = {
|
||||
'@context': 'https://schema.org',
|
||||
'@graph': [personJsonLd(), organizationJsonLd()],
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { JSXConvertersFunction } from '@payloadcms/richtext-lexical/react';
|
||||
import { getCachedPayload } from '@/lib/payload';
|
||||
import type { Post, Media } from '@/payload-types';
|
||||
import { absoluteUrl } from '@/lib/site';
|
||||
import { getFlags } from '@/lib/flags';
|
||||
import {
|
||||
blogPostingJsonLd,
|
||||
breadcrumbJsonLd,
|
||||
@@ -112,6 +113,15 @@ export default async function PostPage(
|
||||
{ params }: { params: Promise<{ slug: string }> },
|
||||
) {
|
||||
const { slug } = await params;
|
||||
|
||||
/*
|
||||
* Flags read at this route's own declared floor, so gating costs it nothing.
|
||||
* Checked before the post is fetched: a dark blog should not query Payload.
|
||||
*/
|
||||
const flags = await getFlags(3600);
|
||||
if (flags.blog !== true) notFound();
|
||||
const namedAuthor = flags.about_page === true;
|
||||
|
||||
const post = await findPost(slug);
|
||||
if (!post) notFound();
|
||||
|
||||
@@ -120,10 +130,15 @@ export default async function PostPage(
|
||||
|
||||
const jsonLd = {
|
||||
'@context': 'https://schema.org',
|
||||
/*
|
||||
* The Person entity is anchored at /about#tudor, so it is declared only
|
||||
* when that page exists. Claiming an author whose URL 404s is a worse
|
||||
* signal than attributing the post to the publisher.
|
||||
*/
|
||||
'@graph': [
|
||||
blogPostingJsonLd(summary),
|
||||
blogPostingJsonLd(summary, { namedAuthor }),
|
||||
breadcrumbJsonLd(summary),
|
||||
personJsonLd(),
|
||||
...(namedAuthor ? [personJsonLd()] : []),
|
||||
organizationJsonLd(),
|
||||
],
|
||||
};
|
||||
@@ -142,7 +157,10 @@ export default async function PostPage(
|
||||
<h1 className={styles.heading}>{summary.title}</h1>
|
||||
|
||||
<p className={styles.byline}>
|
||||
By <Link href="/about" className={styles.link}>Tudor</Link>
|
||||
{/* Unlinked while about_page is dark; the flags are independent. */}
|
||||
By {namedAuthor
|
||||
? <Link href="/about" className={styles.link}>Tudor</Link>
|
||||
: 'Tudor'}
|
||||
{' · '}
|
||||
<time dateTime={summary.publishedAt}>
|
||||
{new Date(summary.publishedAt).toLocaleDateString('en-GB', {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import type { Metadata } from 'next';
|
||||
import Link from 'next/link';
|
||||
import { notFound } from 'next/navigation';
|
||||
import { getCachedPayload } from '@/lib/payload';
|
||||
import { absoluteUrl } from '@/lib/site';
|
||||
import { getFlags } from '@/lib/flags';
|
||||
import styles from './Blog.module.css';
|
||||
|
||||
/*
|
||||
@@ -31,6 +33,9 @@ function formatDate(value: string) {
|
||||
}
|
||||
|
||||
export default async function BlogIndexPage() {
|
||||
const flags = await getFlags();
|
||||
if (flags.blog !== true) notFound();
|
||||
|
||||
const payload = await getCachedPayload();
|
||||
const { docs } = await payload.find({
|
||||
collection: 'posts',
|
||||
@@ -48,7 +53,11 @@ export default async function BlogIndexPage() {
|
||||
<p className={styles.standfirst}>
|
||||
What school performance data shows, what it doesn't, and how to
|
||||
read it without being misled. Written by{' '}
|
||||
<Link href="/about" className={styles.link}>Tudor</Link>.
|
||||
{/* Plain text when about_page is dark: the two flags are
|
||||
independent, so this link would otherwise point at a 404. */}
|
||||
{flags.about_page === true
|
||||
? <Link href="/about" className={styles.link}>Tudor</Link>
|
||||
: 'Tudor'}.
|
||||
</p>
|
||||
</header>
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { getCachedPayload } from '@/lib/payload';
|
||||
import { absoluteUrl } from '@/lib/site';
|
||||
import { getFlags } from '@/lib/flags';
|
||||
|
||||
/*
|
||||
* Dynamic, not ISR.
|
||||
@@ -18,6 +19,11 @@ function escapeXml(value: string): string {
|
||||
}
|
||||
|
||||
export async function GET() {
|
||||
// A dark blog has no feed. 404 rather than an empty channel: an empty feed
|
||||
// is a live feed with nothing in it, which a reader would keep polling.
|
||||
const flags = await getFlags();
|
||||
if (flags.blog !== true) return new Response('Not found', { status: 404 });
|
||||
|
||||
const payload = await getCachedPayload();
|
||||
const { docs } = await payload.find({
|
||||
collection: 'posts',
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
*/
|
||||
import { getCachedPayload } from '@/lib/payload';
|
||||
import { absoluteUrl } from '@/lib/site';
|
||||
import { getFlags } from '@/lib/flags';
|
||||
|
||||
/*
|
||||
* Dynamic, not ISR.
|
||||
@@ -21,18 +22,33 @@ import { absoluteUrl } from '@/lib/site';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function GET() {
|
||||
const payload = await getCachedPayload();
|
||||
const { docs } = await payload.find({
|
||||
collection: 'posts',
|
||||
where: { _status: { equals: 'published' } },
|
||||
sort: '-publishedAt',
|
||||
limit: 500,
|
||||
depth: 0,
|
||||
});
|
||||
/*
|
||||
* A dark page must not be advertised. Submitting a URL that 404s is the one
|
||||
* thing a sitemap is not allowed to do, so each entry is gated on the same
|
||||
* flag that gates the page itself.
|
||||
*
|
||||
* With both flags dark this emits a valid, empty <urlset> rather than a 404:
|
||||
* robots.txt names this sitemap unconditionally, and an empty sitemap is a
|
||||
* well-formed statement that there is nothing here yet.
|
||||
*/
|
||||
const flags = await getFlags();
|
||||
const aboutEnabled = flags.about_page === true;
|
||||
const blogEnabled = flags.blog === true;
|
||||
|
||||
// Only query Payload when the blog is actually being advertised.
|
||||
const docs = blogEnabled
|
||||
? (await (await getCachedPayload()).find({
|
||||
collection: 'posts',
|
||||
where: { _status: { equals: 'published' } },
|
||||
sort: '-publishedAt',
|
||||
limit: 500,
|
||||
depth: 0,
|
||||
})).docs
|
||||
: [];
|
||||
|
||||
const urls: Array<{ loc: string; lastmod: string | null }> = [
|
||||
{ loc: absoluteUrl('/about'), lastmod: null },
|
||||
{ loc: absoluteUrl('/blog'), lastmod: null },
|
||||
...(aboutEnabled ? [{ loc: absoluteUrl('/about'), lastmod: null }] : []),
|
||||
...(blogEnabled ? [{ loc: absoluteUrl('/blog'), lastmod: null }] : []),
|
||||
...docs.map((post) => ({
|
||||
loc: absoluteUrl(`/blog/${post.slug}`),
|
||||
lastmod: new Date(String(post.updatedAt ?? post.publishedAt)).toISOString(),
|
||||
|
||||
@@ -7,6 +7,7 @@ import { ComparisonToast } from '@/components/ComparisonToast';
|
||||
import { RouteTrail } from '@/components/RouteTrail';
|
||||
import { ComparisonProvider } from '@/context/ComparisonProvider';
|
||||
import { SITE_URL } from '@/lib/site';
|
||||
import { getFlags } from '@/lib/flags';
|
||||
import './globals.css';
|
||||
|
||||
// Manrope carries headings and key messaging — the guideline's "friendly,
|
||||
@@ -58,14 +59,32 @@ export const metadata: Metadata = {
|
||||
authors: [{ name: 'schoolcompare' }],
|
||||
manifest: '/manifest.json',
|
||||
// No `icons` key on purpose: setting it here would override the file
|
||||
// conventions. app/icon.svg and app/apple-icon.tsx are the source, and
|
||||
// app/opengraph-image.tsx supplies og:image and twitter:image.
|
||||
// conventions. app/icon.png and app/apple-icon.png are the source.
|
||||
//
|
||||
// og:image and twitter:image are NOT inherited from
|
||||
// app/opengraph-image.tsx — see the note on openGraph.images below. The
|
||||
// icon conventions do reach these pages; the opengraph-image one does not.
|
||||
metadataBase: new URL(SITE_URL),
|
||||
openGraph: {
|
||||
type: 'website',
|
||||
title: 'Compare Schools Side by Side | schoolcompare',
|
||||
description:
|
||||
'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place.',
|
||||
/*
|
||||
* Declared, not inherited.
|
||||
*
|
||||
* app/opengraph-image.tsx is a metadata file convention, and it does
|
||||
* attach to routes in the app root segment — _not-found gets an og:image
|
||||
* from it. It does not reach the site's pages, which live in the
|
||||
* (frontend) route group whose own layout.tsx is a root layout. Staging
|
||||
* served og:title, og:description, og:url, og:site_name and og:type with
|
||||
* no og:image at all, so every link pasted into a chat rendered bare.
|
||||
*
|
||||
* The file stays at the app root: /robots.txt and /icon.png depend on it
|
||||
* being there, and moving it is what broke those before. This points at
|
||||
* the route it generates instead. metadataBase makes it absolute.
|
||||
*/
|
||||
images: ['/opengraph-image'],
|
||||
url: SITE_URL,
|
||||
siteName: 'schoolcompare',
|
||||
},
|
||||
@@ -75,14 +94,34 @@ export const metadata: Metadata = {
|
||||
title: 'Compare Schools Side by Side | schoolcompare',
|
||||
description:
|
||||
'Put five English schools on one screen — SATs, GCSE results, Ofsted grades, and how close you had to live to get a place.',
|
||||
// The card is summary_large_image; claiming that and supplying no image
|
||||
// is worse than claiming a summary card.
|
||||
images: ['/opengraph-image'],
|
||||
},
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
/*
|
||||
* The footer's About and Blog links are flagged, which makes this the one
|
||||
* place on the site that reads a flag on every route.
|
||||
*
|
||||
* 604800 is deliberate and load-bearing: it is the revalidate every SEO route
|
||||
* here already declares. Next pins a route to the LOWEST revalidate among its
|
||||
* fetches, so reading flags at the 300s default would drop the whole school
|
||||
* and place corpus from a weekly cache to a 5-minute one — a large origin-load
|
||||
* regression to hide two footer links.
|
||||
*
|
||||
* The cost is latency in one direction only. The pages themselves read the
|
||||
* same flags at their own floors and flip within minutes; the footer links
|
||||
* follow within a week. Turning a feature on early therefore shows the page
|
||||
* before its footer link, which is harmless. Turning one off leaves a link to
|
||||
* a 404 until the cache turns over, so a rollback that matters wants a purge.
|
||||
*/
|
||||
export default async function RootLayout({
|
||||
children,
|
||||
}: Readonly<{
|
||||
children: React.ReactNode;
|
||||
}>) {
|
||||
const flags = await getFlags(604800);
|
||||
return (
|
||||
// The font variable classes must sit on <html>, not <body>. globals.css
|
||||
// declares --font-display on :root as var(--font-manrope) and --font-ui as
|
||||
@@ -126,7 +165,10 @@ export default function RootLayout({
|
||||
{children}
|
||||
</main>
|
||||
<ComparisonToast />
|
||||
<Footer />
|
||||
<Footer
|
||||
aboutEnabled={flags.about_page === true}
|
||||
blogEnabled={flags.blog === true}
|
||||
/>
|
||||
</ComparisonProvider>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
import { fetchSchoolDetails, fetchSchools, fetchNationalAverages } from '@/lib/api';
|
||||
import { notFound, redirect } from 'next/navigation';
|
||||
import { SchoolDetailShell } from '@/components/school/SchoolDetailShell';
|
||||
import { NearbyPlaces } from '@/components/school/NearbyPlaces';
|
||||
import { schoolBreadcrumbJsonLd, type SchoolPlace } from '@/lib/jsonld';
|
||||
import { PrimarySchoolSections } from '@/components/school/PrimarySchoolSections';
|
||||
import { SecondarySchoolSections } from '@/components/school/SecondarySchoolSections';
|
||||
import {
|
||||
@@ -149,6 +151,10 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
}
|
||||
|
||||
const { school_info, yearly_data, absence_data, ofsted, census, admissions, admissions_history, admission_distance, deprivation, finance, destinations } = data;
|
||||
// Absent on an older API build; the module and the trail both degrade to
|
||||
// nothing rather than throwing, which is how this shipped without a
|
||||
// lockstep deploy of the two images.
|
||||
const places: SchoolPlace[] = data.places ?? [];
|
||||
|
||||
// Redirect bare URN to canonical slug URL
|
||||
const canonicalSlug = schoolUrl(urn, school_info.school_name).replace('/school/', '');
|
||||
@@ -185,10 +191,19 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
const primaryNavItems = buildNavItems(primaryFlags, navInput);
|
||||
const secondaryNavItems = buildSecondaryNavItems(secondaryFlags, navInput);
|
||||
|
||||
// Generate JSON-LD structured data for SEO
|
||||
/*
|
||||
* `School`, not `EducationalOrganization`.
|
||||
*
|
||||
* Both are valid, but EducationalOrganization is the parent type covering
|
||||
* universities, training providers and nurseries alike. School is the
|
||||
* specific one, and a type that says what the page is about is the whole
|
||||
* point of declaring it. Google's own guidance treats the narrower type as
|
||||
* the correct choice where it applies.
|
||||
*/
|
||||
const structuredData = {
|
||||
'@context': 'https://schema.org',
|
||||
'@type': 'EducationalOrganization',
|
||||
'@graph': [{
|
||||
'@type': 'School',
|
||||
name: school_info.school_name,
|
||||
identifier: school_info.urn.toString(),
|
||||
...(school_info.address && {
|
||||
@@ -210,6 +225,15 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
...(school_info.school_type && {
|
||||
additionalType: school_info.school_type,
|
||||
}),
|
||||
},
|
||||
// The trail the page sits at the end of. School pages carried no
|
||||
// breadcrumb at all, while every place page already emitted one.
|
||||
schoolBreadcrumbJsonLd({
|
||||
name: school_info.school_name,
|
||||
url: `/school/${slug}`,
|
||||
places,
|
||||
}),
|
||||
],
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -264,6 +288,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
/>
|
||||
</SchoolDetailShell>
|
||||
)}
|
||||
<NearbyPlaces places={places} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -10,7 +10,17 @@
|
||||
import { LogoMark } from './Logo';
|
||||
import styles from './Footer.module.css';
|
||||
|
||||
export function Footer() {
|
||||
/**
|
||||
* Both default to false so a caller that forgets a prop hides the link rather
|
||||
* than pointing it at a page that 404s. Same reasoning as backend/flags.py:
|
||||
* "Every flag defaults to False."
|
||||
*/
|
||||
interface FooterProps {
|
||||
aboutEnabled?: boolean;
|
||||
blogEnabled?: boolean;
|
||||
}
|
||||
|
||||
export function Footer({ aboutEnabled = false, blogEnabled = false }: FooterProps = {}) {
|
||||
const currentYear = new Date().getFullYear();
|
||||
|
||||
return (
|
||||
@@ -94,17 +104,26 @@ export function Footer() {
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div className={styles.section}>
|
||||
<h4 className={styles.sectionTitle}>About</h4>
|
||||
<ul className={styles.links}>
|
||||
{/* The only route to a named human. Deliberately not in the nav:
|
||||
the mobile bottom bar already carries four items, and both of
|
||||
these are lower intent than any of them. Post bylines link
|
||||
here too, which is where a reader actually asks the question. */}
|
||||
<li><a href="/about" className={styles.link}>Who's behind this</a></li>
|
||||
<li><a href="/blog" className={styles.link}>Blog</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
{/* Dropped entirely when both flags are dark, rather than left as an
|
||||
empty heading: shipping dark means the footer renders as it did
|
||||
before the feature existed. */}
|
||||
{(aboutEnabled || blogEnabled) && (
|
||||
<div className={styles.section}>
|
||||
<h4 className={styles.sectionTitle}>About</h4>
|
||||
<ul className={styles.links}>
|
||||
{/* The only route to a named human. Deliberately not in the nav:
|
||||
the mobile bottom bar already carries four items, and both of
|
||||
these are lower intent than any of them. Post bylines link
|
||||
here too, which is where a reader actually asks the question. */}
|
||||
{aboutEnabled && (
|
||||
<li><a href="/about" className={styles.link}>Who's behind this</a></li>
|
||||
)}
|
||||
{blogEnabled && (
|
||||
<li><a href="/blog" className={styles.link}>Blog</a></li>
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className={styles.bottom}>
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
/* Tokens only — the same vocabulary schoolSections.module.css uses, so the
|
||||
module follows both themes without a rule of its own. No hardcoded colour
|
||||
appears here; darkThemeSafety asserts that across the codebase. */
|
||||
|
||||
.section {
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
/* Matches .sectionTitle in schoolSections.module.css, including the brand
|
||||
rule before the text, so this reads as one more section of the page
|
||||
rather than a footer bolted underneath it. */
|
||||
.heading {
|
||||
font-size: 1.125rem;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
margin-bottom: 0.875rem;
|
||||
padding-bottom: 0.5rem;
|
||||
border-bottom: 2px solid var(--border);
|
||||
font-family: var(--font-display);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.375rem;
|
||||
}
|
||||
|
||||
.heading::before {
|
||||
content: "";
|
||||
display: inline-block;
|
||||
width: 3px;
|
||||
height: 1em;
|
||||
background: var(--brand);
|
||||
border-radius: 2px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.5rem 1.25rem;
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.link {
|
||||
color: var(--brand-strong);
|
||||
font-weight: 500;
|
||||
text-decoration: underline;
|
||||
text-underline-offset: 2px;
|
||||
}
|
||||
|
||||
.link:hover {
|
||||
text-decoration-thickness: 2px;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import Link from 'next/link';
|
||||
import type { SchoolPlace, SchoolPhasePage } from '@/lib/jsonld';
|
||||
import styles from './NearbyPlaces.module.css';
|
||||
|
||||
/**
|
||||
* Links from a school page into the location layer.
|
||||
*
|
||||
* This exists for a structural reason rather than a decorative one. Before
|
||||
* it, the only anchor on a school page pointed at the school's own website,
|
||||
* so the ~27k pages that carry most of the site's inbound authority passed it
|
||||
* straight off-site and none of it reached the place pages. These links are
|
||||
* what circulate it instead.
|
||||
*
|
||||
* Every entry comes from the place registry via the API, so a link is only
|
||||
* ever offered for a page that exists: a place below the publish threshold is
|
||||
* absent from the registry and therefore absent here.
|
||||
*/
|
||||
|
||||
/** Narrowest first: a reader on a school page wants its town before its
|
||||
* county. The API orders widest-first because that is what the breadcrumb
|
||||
* reads, so the two orders are deliberately different. */
|
||||
const ORDER: Record<string, number> = {
|
||||
town: 0, locality: 0, outcode: 1, authority: 2,
|
||||
};
|
||||
|
||||
function label(place: SchoolPlace): string {
|
||||
const noun = place.count === 1 ? 'school' : 'schools';
|
||||
const preposition = place.kind === 'outcode' ? 'near' : 'in';
|
||||
return `${place.count} ${noun} ${preposition} ${place.name}`;
|
||||
}
|
||||
|
||||
/** "22 primary schools in Brentwood" — the phrasing the query itself uses. */
|
||||
function phaseLabel(place: SchoolPlace, page: SchoolPhasePage): string {
|
||||
const noun = page.count === 1 ? 'school' : 'schools';
|
||||
return `${page.count} ${page.phase} ${noun} in ${place.name}`;
|
||||
}
|
||||
|
||||
export function NearbyPlaces({ places }: { places: SchoolPlace[] }) {
|
||||
if (places.length === 0) return null;
|
||||
|
||||
const sorted = [...places].sort(
|
||||
(a, b) => (ORDER[a.kind] ?? 9) - (ORDER[b.kind] ?? 9),
|
||||
);
|
||||
|
||||
return (
|
||||
<section className={styles.section} aria-labelledby="nearby-places">
|
||||
<h2 id="nearby-places" className={styles.heading}>More schools near here</h2>
|
||||
<ul className={styles.list}>
|
||||
{sorted.flatMap((place) => [
|
||||
<li key={`${place.kind}:${place.slug}`}>
|
||||
<Link href={place.url} className={styles.link}>{label(place)}</Link>
|
||||
</li>,
|
||||
/* Immediately after its own place, so "22 primary schools in
|
||||
Brentwood" reads as part of Brentwood rather than as an
|
||||
unrelated link further down the row. */
|
||||
...place.phases.map((page) => (
|
||||
<li key={`${place.kind}:${place.slug}:${page.phase}`}>
|
||||
<Link href={page.url} className={styles.link}>
|
||||
{phaseLabel(place, page)}
|
||||
</Link>
|
||||
</li>
|
||||
)),
|
||||
])}
|
||||
</ul>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -4,6 +4,26 @@ The blog is Payload CMS, running inside the Next.js app. There is no separate
|
||||
service and no second deploy. Writing a post is done in the browser and takes
|
||||
effect on the live site within seconds.
|
||||
|
||||
## Before any of this: the flags
|
||||
|
||||
`/blog` and `/about` are behind feature flags (`blog` and `about_page`), and
|
||||
every flag in this system starts off. While `blog` is dark, `/blog`, every post
|
||||
page and the RSS feed return 404, and neither appears in the sitemap or the
|
||||
footer. Posts still save normally, because `/admin` is deliberately **not**
|
||||
flagged: you have to be able to write a post before there is anything worth
|
||||
switching on.
|
||||
|
||||
So a new post published to a dark blog is invisible, and that is working as
|
||||
intended, not a bug. Flip the flag in Unleash when the content is ready.
|
||||
Staging and production hold their own values (`development` and `production`
|
||||
environments), so you can light it on staging first. The page follows a flip
|
||||
within five minutes; the footer link takes up to a week, because it renders in
|
||||
the root layout and is cached at the same weekly floor as the school corpus.
|
||||
|
||||
Both flags are temporary scaffolding, like every flag here: a test starts
|
||||
failing once one is older than 90 days, at which point either the feature is
|
||||
permanent and the flag comes out, or it was never going to ship.
|
||||
|
||||
## Signing in
|
||||
|
||||
`https://www.schoolcompare.co.uk/admin`, one account, no registration. If you
|
||||
|
||||
@@ -311,26 +311,6 @@ export async function fetchDataInfo(
|
||||
return handleResponse<DataInfoResponse>(response);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Client-Side Fetcher (for SWR)
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Generic fetcher function for use with SWR
|
||||
* @example
|
||||
* ```tsx
|
||||
* const { data, error } = useSWR('/api/schools', fetcher);
|
||||
* ```
|
||||
*/
|
||||
export async function fetcher<T>(url: string): Promise<T> {
|
||||
// If it's already a full URL, use it directly
|
||||
// Otherwise, prepend the API_BASE_URL
|
||||
const fullUrl = url.startsWith('http') ? url : `${API_BASE_URL}${url.startsWith('/') ? url : `/${url}`}`;
|
||||
|
||||
const response = await fetch(fullUrl);
|
||||
return handleResponse<T>(response);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Geocoding API
|
||||
// ============================================================================
|
||||
@@ -396,10 +376,3 @@ export function calculateDistance(
|
||||
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
||||
return R * c;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert kilometers to miles
|
||||
*/
|
||||
export function kmToMiles(km: number): number {
|
||||
return km * 0.621371;
|
||||
}
|
||||
+13
-3
@@ -26,11 +26,21 @@ export const FLAGS_REVALIDATE = 300;
|
||||
const API = process.env.FASTAPI_URL || process.env.NEXT_PUBLIC_API_URL
|
||||
|| 'http://localhost:8000/api';
|
||||
|
||||
/** Every flag and its value. Never throws: an unreadable flag is a dark one. */
|
||||
export async function getFlags(): Promise<Flags> {
|
||||
/**
|
||||
* Every flag and its value. Never throws: an unreadable flag is a dark one.
|
||||
*
|
||||
* `revalidate` is the caller's, because reading flags pins the whole route to
|
||||
* the lowest revalidate among its fetches. Every SEO route here declares
|
||||
* 604800; gating one at the 300s default would drop it from a weekly cache to
|
||||
* a 5-minute one. Pass the route's own floor and gating costs it nothing.
|
||||
*
|
||||
* The trade is flag-flip latency: a route that revalidates weekly takes up to
|
||||
* a week to notice a flip. Pass a smaller number where a flip must land fast.
|
||||
*/
|
||||
export async function getFlags(revalidate: number = FLAGS_REVALIDATE): Promise<Flags> {
|
||||
try {
|
||||
const res = await fetch(`${API}/flags`, {
|
||||
next: { revalidate: FLAGS_REVALIDATE },
|
||||
next: { revalidate },
|
||||
});
|
||||
if (!res.ok) return {};
|
||||
return await res.json();
|
||||
|
||||
@@ -44,19 +44,100 @@ interface PostSummary {
|
||||
* References the Person and Organization by @id rather than repeating them, so
|
||||
* search engines resolve every post and the About page to the one author
|
||||
* entity. Repeating the shape would declare several people with one name.
|
||||
*
|
||||
* `namedAuthor` is the about_page flag. The Person entity is anchored at
|
||||
* /about#tudor, and that URL 404s while the flag is dark, so a post published
|
||||
* in that state must not claim it: an author @id resolving to nothing is a
|
||||
* worse signal than no named author. It falls back to the publisher, which is
|
||||
* always live. The parameter is required rather than defaulted because every
|
||||
* call site has the flag to hand and the wrong default is silent.
|
||||
*/
|
||||
export function blogPostingJsonLd(post: PostSummary) {
|
||||
export function blogPostingJsonLd(
|
||||
post: PostSummary,
|
||||
{ namedAuthor }: { namedAuthor: boolean },
|
||||
) {
|
||||
return {
|
||||
'@type': 'BlogPosting',
|
||||
headline: post.title,
|
||||
description: post.excerpt,
|
||||
url: absoluteUrl(`/blog/${post.slug}`),
|
||||
datePublished: post.publishedAt,
|
||||
author: { '@id': `${SITE_URL}/about#tudor` },
|
||||
author: {
|
||||
'@id': namedAuthor ? `${SITE_URL}/about#tudor` : `${SITE_URL}#organization`,
|
||||
},
|
||||
publisher: { '@id': `${SITE_URL}#organization` },
|
||||
} as const;
|
||||
}
|
||||
|
||||
/**
|
||||
* A place the location layer publishes a page for, as the school API reports
|
||||
* it. `count` is what lets a link say "All 37 schools in Brentwood" rather
|
||||
* than "click here".
|
||||
*/
|
||||
export interface SchoolPhasePage {
|
||||
phase: string;
|
||||
count: number;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface SchoolPlace {
|
||||
kind: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
count: number;
|
||||
url: string;
|
||||
/**
|
||||
* The phase variants this school is actually listed on: usually one, two
|
||||
* for an all-through school, none for an outcode, which publishes no phase
|
||||
* route. Decided by the place registry, never re-derived here.
|
||||
*/
|
||||
phases: SchoolPhasePage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The trail a school page sits at the end of: Schools → authority → town.
|
||||
*
|
||||
* Only authority and town/locality appear. An outcode is a useful link in the
|
||||
* module beside this — a parent does search "schools near CM15" — but it is
|
||||
* not a step anyone navigates through, and a breadcrumb that claims otherwise
|
||||
* describes a hierarchy the site does not have.
|
||||
*
|
||||
* Levels are skipped rather than faked. A school whose town falls below the
|
||||
* publish threshold has no town page, so the trail closes over the gap; the
|
||||
* alternative is a breadcrumb linking to a 404.
|
||||
*/
|
||||
export function schoolBreadcrumbJsonLd(
|
||||
school: { name: string; url: string; places: SchoolPlace[] },
|
||||
) {
|
||||
/*
|
||||
* Rooted at the homepage, not at /schools. There is no /schools index page
|
||||
* — the location layer is /schools/[place], /schools/authority/[la] and
|
||||
* /schools/near/[outcode], with nothing at the bare path — so a trail
|
||||
* starting there would open with a link to a 404.
|
||||
*/
|
||||
const trail: Array<{ name: string; url: string }> = [
|
||||
{ name: 'schoolcompare', url: '/' },
|
||||
];
|
||||
|
||||
const authority = school.places.find((p) => p.kind === 'authority');
|
||||
if (authority) trail.push({ name: authority.name, url: authority.url });
|
||||
|
||||
const town = school.places.find((p) => p.kind === 'town' || p.kind === 'locality');
|
||||
if (town) trail.push({ name: town.name, url: town.url });
|
||||
|
||||
trail.push({ name: school.name, url: school.url });
|
||||
|
||||
return {
|
||||
'@type': 'BreadcrumbList',
|
||||
itemListElement: trail.map((step, index) => ({
|
||||
'@type': 'ListItem',
|
||||
position: index + 1,
|
||||
name: step.name,
|
||||
item: absoluteUrl(step.url),
|
||||
})),
|
||||
} as const;
|
||||
}
|
||||
|
||||
export function breadcrumbJsonLd(post: PostSummary) {
|
||||
return {
|
||||
'@type': 'BreadcrumbList',
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { SchoolPlace } from '@/lib/jsonld';
|
||||
|
||||
/**
|
||||
* TypeScript type definitions for SchoolCompare API
|
||||
* Generated from backend/models.py and backend/schemas.py
|
||||
@@ -346,6 +348,15 @@ export interface SchoolsResponse {
|
||||
|
||||
export interface SchoolDetailsResponse {
|
||||
school_info: School;
|
||||
/**
|
||||
* The published location-layer pages containing this school, widest first.
|
||||
*
|
||||
* Optional because the frontend and backend ship as separate images: a
|
||||
* frontend deployed ahead of the API that serves this must render without
|
||||
* it, not throw. Empty is also a real answer — a school whose town and
|
||||
* authority both fall below the publish threshold has nowhere to link.
|
||||
*/
|
||||
places?: SchoolPlace[];
|
||||
yearly_data: SchoolResult[];
|
||||
absence_data: AbsenceData | null;
|
||||
// Supplementary data (null until Kestra populates)
|
||||
|
||||
@@ -24,9 +24,8 @@ export default buildConfig({
|
||||
typescript: { outputFile: path.resolve(dirname, 'payload-types.ts') },
|
||||
db: postgresAdapter({
|
||||
pool: { connectionString: process.env.DATABASE_URL },
|
||||
// Its own schema, so no pipeline operation on `public` can reach blog
|
||||
// content. scripts/migrate_csv_to_db.py --drop lives in that blast radius,
|
||||
// as does Airflow's metadata. The schema itself is created by the initial
|
||||
// Its own schema separates blog content from pipeline-managed school
|
||||
// tables and Airflow metadata. The schema is created by the initial
|
||||
// migration: schemaName says where tables go, it does not create anything.
|
||||
//
|
||||
// prodMigrations runs pending migrations during server init. Without it a
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Historical standalone CSV utility; not part of the managed Meltano/dbt pipeline.
|
||||
See docs/LEGACY_CODE.md before using it for current school data.
|
||||
|
||||
Data Download Helper Script
|
||||
|
||||
This script provides instructions and utilities for downloading
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Historical standalone CSV utility; not part of the managed Meltano/dbt pipeline.
|
||||
See docs/LEGACY_CODE.md before using it for current school data.
|
||||
|
||||
Fetch real school performance data from UK Government sources.
|
||||
|
||||
This script downloads KS2 (Key Stage 2) primary school data from:
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Geocode all school postcodes and update the database.
|
||||
|
||||
This script should be run as a weekly cron job to ensure all schools
|
||||
have up-to-date latitude/longitude coordinates.
|
||||
|
||||
Usage:
|
||||
python scripts/geocode_schools.py [--force]
|
||||
|
||||
Options:
|
||||
--force Re-geocode all postcodes, even if already geocoded
|
||||
|
||||
Crontab example (run every Sunday at 2am):
|
||||
0 2 * * 0 cd /path/to/school_compare && /path/to/venv/bin/python scripts/geocode_schools.py >> /var/log/geocode_schools.log 2>&1
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from backend.database import SessionLocal
|
||||
from backend.models import School
|
||||
|
||||
|
||||
def geocode_postcodes_bulk(postcodes: list) -> Dict[str, Tuple[float, float]]:
|
||||
"""
|
||||
Geocode postcodes in bulk using postcodes.io API.
|
||||
Returns dict of postcode -> (latitude, longitude).
|
||||
"""
|
||||
results = {}
|
||||
valid_postcodes = [
|
||||
p.strip().upper()
|
||||
for p in postcodes
|
||||
if p and isinstance(p, str) and len(p.strip()) >= 5
|
||||
]
|
||||
valid_postcodes = list(set(valid_postcodes))
|
||||
|
||||
if not valid_postcodes:
|
||||
return results
|
||||
|
||||
batch_size = 100
|
||||
total_batches = (len(valid_postcodes) + batch_size - 1) // batch_size
|
||||
|
||||
for i, batch_start in enumerate(range(0, len(valid_postcodes), batch_size)):
|
||||
batch = valid_postcodes[batch_start : batch_start + batch_size]
|
||||
print(f" Geocoding batch {i + 1}/{total_batches} ({len(batch)} postcodes)...")
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
"https://api.postcodes.io/postcodes",
|
||||
json={"postcodes": batch},
|
||||
timeout=30,
|
||||
)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
for item in data.get("result", []):
|
||||
if item and item.get("result"):
|
||||
pc = item["query"].upper()
|
||||
lat = item["result"].get("latitude")
|
||||
lon = item["result"].get("longitude")
|
||||
if lat and lon:
|
||||
results[pc] = (lat, lon)
|
||||
else:
|
||||
print(f" Warning: API returned status {response.status_code}")
|
||||
except Exception as e:
|
||||
print(f" Warning: Geocoding batch failed: {e}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def geocode_schools(force: bool = False) -> None:
|
||||
"""
|
||||
Geocode all schools in the database.
|
||||
|
||||
Args:
|
||||
force: If True, re-geocode all postcodes even if already geocoded
|
||||
"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"School Geocoding Job - {datetime.now().isoformat()}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
db = SessionLocal()
|
||||
|
||||
try:
|
||||
# Get schools that need geocoding
|
||||
if force:
|
||||
schools = db.query(School).filter(School.postcode.isnot(None)).all()
|
||||
print(f"Force mode: Processing all {len(schools)} schools with postcodes")
|
||||
else:
|
||||
schools = db.query(School).filter(
|
||||
School.postcode.isnot(None),
|
||||
(School.latitude.is_(None)) | (School.longitude.is_(None))
|
||||
).all()
|
||||
print(f"Found {len(schools)} schools without coordinates")
|
||||
|
||||
if not schools:
|
||||
print("No schools to geocode. Exiting.")
|
||||
return
|
||||
|
||||
# Extract unique postcodes
|
||||
postcodes = list(set(
|
||||
s.postcode.strip().upper()
|
||||
for s in schools
|
||||
if s.postcode
|
||||
))
|
||||
print(f"Unique postcodes to geocode: {len(postcodes)}")
|
||||
|
||||
# Geocode in bulk
|
||||
print("\nGeocoding postcodes...")
|
||||
geocoded = geocode_postcodes_bulk(postcodes)
|
||||
print(f"Successfully geocoded: {len(geocoded)} postcodes")
|
||||
|
||||
# Update database
|
||||
print("\nUpdating database...")
|
||||
updated_count = 0
|
||||
failed_count = 0
|
||||
|
||||
for school in schools:
|
||||
if not school.postcode:
|
||||
continue
|
||||
|
||||
pc_upper = school.postcode.strip().upper()
|
||||
coords = geocoded.get(pc_upper)
|
||||
|
||||
if coords:
|
||||
school.latitude = coords[0]
|
||||
school.longitude = coords[1]
|
||||
updated_count += 1
|
||||
else:
|
||||
failed_count += 1
|
||||
|
||||
db.commit()
|
||||
|
||||
print(f"\nResults:")
|
||||
print(f" - Updated: {updated_count} schools")
|
||||
print(f" - Failed (invalid/not found): {failed_count} postcodes")
|
||||
|
||||
# Summary stats
|
||||
total_with_coords = db.query(School).filter(
|
||||
School.latitude.isnot(None),
|
||||
School.longitude.isnot(None)
|
||||
).count()
|
||||
total_schools = db.query(School).count()
|
||||
|
||||
print(f"\nDatabase summary:")
|
||||
print(f" - Total schools: {total_schools}")
|
||||
print(f" - Schools with coordinates: {total_with_coords}")
|
||||
print(f" - Coverage: {100*total_with_coords/total_schools:.1f}%")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during geocoding: {e}")
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Geocoding job completed - {datetime.now().isoformat()}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Geocode school postcodes and update database"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Re-geocode all postcodes, even if already geocoded"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
geocode_schools(force=args.force)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CLI script for manual database migration.
|
||||
|
||||
Usage:
|
||||
python scripts/migrate_csv_to_db.py [--drop] [--geocode]
|
||||
|
||||
Options:
|
||||
--drop Drop existing tables before migration (full reimport)
|
||||
--geocode Geocode postcodes (requires network access)
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import argparse
|
||||
|
||||
from backend.config import settings
|
||||
from backend.database import Base, engine, init_db, set_db_schema_version
|
||||
from backend.migration import load_csv_data, migrate_data, run_full_migration
|
||||
from backend.version import SCHEMA_VERSION
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Migrate CSV data to PostgreSQL database"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--drop", action="store_true", help="Drop existing tables before migration"
|
||||
)
|
||||
parser.add_argument("--geocode", action="store_true", help="Geocode postcodes")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("School Data Migration: CSV -> PostgreSQL")
|
||||
print("=" * 60)
|
||||
print(f"\nDatabase: {settings.database_url.split('@')[-1]}")
|
||||
print(f"Data directory: {settings.data_dir}")
|
||||
print(f"Target schema version: {SCHEMA_VERSION}")
|
||||
|
||||
if args.drop:
|
||||
print("\nRunning full migration (drop + reimport)...")
|
||||
success = run_full_migration(geocode=args.geocode)
|
||||
else:
|
||||
print("\nCreating tables (preserving existing data)...")
|
||||
init_db()
|
||||
print("\nLoading CSV data...")
|
||||
df = load_csv_data(settings.data_dir)
|
||||
if df.empty:
|
||||
print("No data found to migrate!")
|
||||
return 1
|
||||
migrate_data(df, geocode=args.geocode)
|
||||
success = True
|
||||
|
||||
if success:
|
||||
# Ensure schema_version table exists
|
||||
init_db()
|
||||
set_db_schema_version(SCHEMA_VERSION)
|
||||
print(f"\nSchema version set to {SCHEMA_VERSION}")
|
||||
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user