Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d0100cce69 |
@@ -51,14 +51,11 @@ jobs:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt pytest "httpx<0.28"
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
- name: Import smoke test
|
||||
run: python -c "from backend.app import app; print('backend imports OK')"
|
||||
|
||||
- name: Backend unit tests
|
||||
run: python -m pytest backend/tests -q
|
||||
|
||||
build-backend:
|
||||
name: Build Backend (no push)
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
name: PR Comment Agent
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
ai-fixup:
|
||||
name: Claude Fix-up (@claude comment)
|
||||
runs-on: ubuntu-latest
|
||||
# Only PR comments, only from the repo owner, only when addressed to @claude.
|
||||
# The owner guard matters: the job pushes code to the PR branch.
|
||||
if: >-
|
||||
gitea.event.issue.pull_request &&
|
||||
startsWith(gitea.event.comment.body, '@claude') &&
|
||||
gitea.event.comment.user.login == gitea.repository_owner
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install Claude Code
|
||||
run: npm install -g @anthropic-ai/claude-code
|
||||
|
||||
- name: Apply the requested fix-up
|
||||
env:
|
||||
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITEA_SERVER_URL: ${{ gitea.server_url }}
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
PR_NUMBER: ${{ gitea.event.issue.number }}
|
||||
COMMENT_BODY: ${{ gitea.event.comment.body }}
|
||||
# Claude Code runs as root inside the runner container; this flag
|
||||
# acknowledges the container *is* the sandbox.
|
||||
IS_SANDBOX: "1"
|
||||
run: python scripts/ci/ai_fixup.py
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
venv
|
||||
__pycache__/
|
||||
backend/__pycache__
|
||||
|
||||
+11
-31
@@ -33,7 +33,7 @@ from .data_loader import (
|
||||
)
|
||||
from .data_loader import get_data_info as get_db_info
|
||||
from .schemas import METRIC_DEFINITIONS, RANKING_COLUMNS, SCHOOL_COLUMNS
|
||||
from .utils import clean_for_json, convert_to_native
|
||||
from .utils import clean_for_json
|
||||
|
||||
# Values to exclude from filter dropdowns (empty strings, non-applicable labels)
|
||||
EXCLUDED_FILTER_VALUES = {"", "Not applicable", "Does not apply"}
|
||||
@@ -416,17 +416,10 @@ async def get_schools(
|
||||
df_latest = df_latest[df_latest["gender"].str.lower() == gender.lower()]
|
||||
if admissions_policy:
|
||||
df_latest = df_latest[df_latest["admissions_policy"].str.lower() == admissions_policy.lower()]
|
||||
# GIAS OfficialSixthForm flag (dim_school.has_sixth_form). NULL (flag not
|
||||
# yet populated by the pipeline) is treated as "no sixth form".
|
||||
if has_sixth_form in ("yes", "no"):
|
||||
if "has_sixth_form" in df_latest.columns:
|
||||
flag = df_latest["has_sixth_form"].eq(True)
|
||||
else: # Defensive fallback only — data_loader now always synthesizes
|
||||
# has_sixth_form as NULL when the DB predates the pipeline re-run,
|
||||
# so this branch shouldn't normally trigger. Falls back to age
|
||||
# range if the column is somehow absent anyway.
|
||||
flag = df_latest["age_range"].str.contains("18", na=False)
|
||||
df_latest = df_latest[flag if has_sixth_form == "yes" else ~flag]
|
||||
if has_sixth_form == "yes":
|
||||
df_latest = df_latest[df_latest["age_range"].str.contains("18", na=False)]
|
||||
elif has_sixth_form == "no":
|
||||
df_latest = df_latest[~df_latest["age_range"].str.contains("18", na=False)]
|
||||
|
||||
# Include key result metrics for display on cards
|
||||
location_cols = ["latitude", "longitude"]
|
||||
@@ -579,7 +572,7 @@ async def get_school_details(request: Request, urn: int):
|
||||
# Get latest info for the school
|
||||
latest = school_data.iloc[-1]
|
||||
|
||||
# Fetch supplementary data (Ofsted, admissions, etc.)
|
||||
# Fetch supplementary data (Ofsted, Parent View, admissions, etc.)
|
||||
from .database import SessionLocal
|
||||
supplementary = {}
|
||||
try:
|
||||
@@ -589,13 +582,8 @@ async def get_school_details(request: Request, urn: int):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Schools with no performance rows (post-16 institutions, PRUs, new
|
||||
# schools) carry NaN in every LEFT-JOINed numeric column; NaN reaching
|
||||
# JSONResponse raises ValueError, so school_info needs the same
|
||||
# conversion yearly_data gets from clean_for_json.
|
||||
school_info = {
|
||||
k: convert_to_native(v)
|
||||
for k, v in {
|
||||
return {
|
||||
"school_info": {
|
||||
"urn": urn,
|
||||
"school_name": latest.get("school_name", ""),
|
||||
"local_authority": latest.get("local_authority", ""),
|
||||
@@ -603,8 +591,6 @@ async def get_school_details(request: Request, urn: int):
|
||||
"address": latest.get("address", ""),
|
||||
"religious_denomination": latest.get("religious_denomination", ""),
|
||||
"age_range": latest.get("age_range", ""),
|
||||
"has_sixth_form": latest.get("has_sixth_form"),
|
||||
"status": latest.get("status"),
|
||||
"latitude": latest.get("latitude"),
|
||||
"longitude": latest.get("longitude"),
|
||||
"phase": latest.get("phase"),
|
||||
@@ -615,14 +601,11 @@ async def get_school_details(request: Request, urn: int):
|
||||
"total_pupils": latest.get("gias_total_pupils"),
|
||||
"trust_name": latest.get("trust_name"),
|
||||
"gender": latest.get("gender"),
|
||||
}.items()
|
||||
}
|
||||
|
||||
return {
|
||||
"school_info": school_info,
|
||||
},
|
||||
"yearly_data": clean_for_json(school_data),
|
||||
# Supplementary data (null if not yet populated by Kestra)
|
||||
"ofsted": supplementary.get("ofsted"),
|
||||
"parent_view": supplementary.get("parent_view"),
|
||||
"census": supplementary.get("census"),
|
||||
"admissions": supplementary.get("admissions"),
|
||||
"admissions_history": supplementary.get("admissions_history") or [],
|
||||
@@ -851,10 +834,7 @@ async def get_rankings(
|
||||
request: Request,
|
||||
metric: str = Query("rwm_expected_pct", description="Metric to rank by", max_length=50),
|
||||
year: Optional[int] = Query(
|
||||
None,
|
||||
description="Academic year code, e.g. 201819 (defaults to most recent)",
|
||||
ge=2000,
|
||||
le=210100,
|
||||
None, description="Specific year (defaults to most recent)", ge=2000, le=2100
|
||||
),
|
||||
limit: int = Query(20, ge=1, le=100, description="Number of schools to return"),
|
||||
local_authority: Optional[str] = Query(
|
||||
|
||||
+29
-126
@@ -3,57 +3,21 @@ Data loading module — reads from marts.* tables built by dbt.
|
||||
Provides efficient queries with caching.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional, Dict, Tuple, List
|
||||
import requests
|
||||
from sqlalchemy import text
|
||||
import sqlalchemy.exc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import settings
|
||||
from .database import SessionLocal, engine
|
||||
from .models import (
|
||||
DimSchool, DimLocation, KS2Performance,
|
||||
FactOfstedInspection, FactAdmissions,
|
||||
FactOfstedInspection, FactParentView, FactAdmissions,
|
||||
FactDeprivation, FactFinance, FactPupilCharacteristics,
|
||||
)
|
||||
from .schemas import SCHOOL_TYPE_MAP
|
||||
from .gias_codes import (
|
||||
ADMISSIONS_POLICY,
|
||||
ESTABLISHMENT_STATUS,
|
||||
PHASE_OF_EDUCATION,
|
||||
RELIGIOUS_CHARACTER,
|
||||
SCHOOL_TYPE,
|
||||
translate,
|
||||
)
|
||||
|
||||
# mart code column -> (API name column, dictionary)
|
||||
_GIAS_CODE_COLUMNS = {
|
||||
"phase_code": ("phase", PHASE_OF_EDUCATION),
|
||||
"school_type_code": ("school_type", SCHOOL_TYPE),
|
||||
"status_code": ("status", ESTABLISHMENT_STATUS),
|
||||
"religious_character_code": ("religious_denomination", RELIGIOUS_CHARACTER),
|
||||
"admissions_policy_code": ("admissions_policy", ADMISSIONS_POLICY),
|
||||
}
|
||||
|
||||
|
||||
def translate_gias_code_columns(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Map GIAS code columns to today's name columns (API contract).
|
||||
|
||||
Runs immediately after pd.read_sql so every downstream consumer —
|
||||
filters, PHASE_GROUPS, payloads, /api/filters — keeps seeing names.
|
||||
DataFrames without the code columns (old schema, test fixtures) pass
|
||||
through unchanged.
|
||||
"""
|
||||
for code_col, (name_col, mapping) in _GIAS_CODE_COLUMNS.items():
|
||||
if code_col in df.columns:
|
||||
df[name_col] = df[code_col].map(lambda c: translate(c, mapping))
|
||||
return df
|
||||
|
||||
|
||||
_postcode_cache: Dict[str, Tuple[float, float]] = {}
|
||||
_typesense_client = None
|
||||
@@ -154,16 +118,14 @@ _MAIN_QUERY = text("""
|
||||
SELECT
|
||||
s.urn,
|
||||
s.school_name,
|
||||
s.phase_code,
|
||||
s.school_type_code,
|
||||
s.phase,
|
||||
s.school_type,
|
||||
s.academy_trust_name AS trust_name,
|
||||
s.academy_trust_uid AS trust_uid,
|
||||
s.religious_character_code,
|
||||
s.religious_character AS religious_denomination,
|
||||
s.gender,
|
||||
s.age_range,
|
||||
s.has_sixth_form,
|
||||
s.status_code,
|
||||
s.admissions_policy_code,
|
||||
s.admissions_policy,
|
||||
s.capacity,
|
||||
s.total_pupils AS gias_total_pupils,
|
||||
s.headteacher_name,
|
||||
@@ -252,92 +214,11 @@ _MAIN_QUERY = text("""
|
||||
ORDER BY s.school_name, p.year
|
||||
""")
|
||||
|
||||
# Fallback used when marts.dim_school predates the has_sixth_form column
|
||||
# (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on this DB).
|
||||
# Keeps the column present as NULL so downstream code — including the
|
||||
# app.py fallback branch — behaves as designed instead of KeyError-ing.
|
||||
_MAIN_QUERY_NO_SIXTH_FORM = text(
|
||||
str(_MAIN_QUERY).replace("s.has_sixth_form,", "NULL AS has_sixth_form,")
|
||||
)
|
||||
assert "NULL AS has_sixth_form" in str(_MAIN_QUERY_NO_SIXTH_FORM), (
|
||||
"expected replacement of 's.has_sixth_form,' to have taken effect"
|
||||
)
|
||||
|
||||
# Fallback used when marts.dim_school predates the GIAS code-dictionary
|
||||
# migration (i.e. the nightly dbt pipeline hasn't rebuilt the mart yet on
|
||||
# this DB, so it still has the old name columns instead of *_code columns).
|
||||
_MAIN_QUERY_LEGACY_NAMES = str(_MAIN_QUERY)
|
||||
_LEGACY_NAME_REPLACEMENTS = [
|
||||
("s.phase_code,", "s.phase,"),
|
||||
("s.school_type_code,", "s.school_type,"),
|
||||
(
|
||||
"s.religious_character_code,",
|
||||
"s.religious_character AS religious_denomination,",
|
||||
),
|
||||
("s.status_code,", "s.status,"),
|
||||
("s.admissions_policy_code,", "s.admissions_policy,"),
|
||||
]
|
||||
for _old, _new in _LEGACY_NAME_REPLACEMENTS:
|
||||
assert _old in _MAIN_QUERY_LEGACY_NAMES, (
|
||||
f"expected {_old!r} to be present in _MAIN_QUERY before replacement"
|
||||
)
|
||||
_MAIN_QUERY_LEGACY_NAMES = _MAIN_QUERY_LEGACY_NAMES.replace(_old, _new)
|
||||
_MAIN_QUERY_LEGACY_NAMES = text(_MAIN_QUERY_LEGACY_NAMES)
|
||||
|
||||
_GIAS_CODE_COLUMN_NAMES = (
|
||||
"phase_code",
|
||||
"school_type_code",
|
||||
"religious_character_code",
|
||||
"status_code",
|
||||
"admissions_policy_code",
|
||||
)
|
||||
|
||||
_MISSING_COLUMN_RE = re.compile(r'column "?(?:s\.)?(\w+)"? does not exist')
|
||||
|
||||
|
||||
def _missing_column_name(exc: Exception) -> Optional[str]:
|
||||
"""Name of the missing column from a psycopg2 UndefinedColumn error.
|
||||
|
||||
Inspects exc.orig (the DBAPI error), whose message names only the
|
||||
offending column — str(exc) also embeds the full SQL statement, which
|
||||
contains every column name and therefore must not be matched against.
|
||||
"""
|
||||
orig = getattr(exc, "orig", None)
|
||||
match = _MISSING_COLUMN_RE.search(str(orig) if orig is not None else str(exc))
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def load_school_data_as_dataframe() -> pd.DataFrame:
|
||||
"""Load all school + KS2 data as a pandas DataFrame."""
|
||||
try:
|
||||
df = pd.read_sql(_MAIN_QUERY, engine)
|
||||
except sqlalchemy.exc.ProgrammingError as exc:
|
||||
missing = _missing_column_name(exc)
|
||||
if missing in _GIAS_CODE_COLUMN_NAMES:
|
||||
logging.getLogger(__name__).warning(
|
||||
"marts predate the GIAS code migration — falling back to "
|
||||
"legacy name-column query: %s",
|
||||
exc,
|
||||
)
|
||||
try:
|
||||
df = pd.read_sql(_MAIN_QUERY_LEGACY_NAMES, engine)
|
||||
except Exception as exc2:
|
||||
print(f"Warning: Could not load school data from marts: {exc2}")
|
||||
return pd.DataFrame()
|
||||
elif missing == "has_sixth_form":
|
||||
logging.getLogger(__name__).warning(
|
||||
"marts.dim_school is missing has_sixth_form (pipeline hasn't "
|
||||
"rebuilt the mart yet on this DB) — retrying without it: %s",
|
||||
exc,
|
||||
)
|
||||
try:
|
||||
df = pd.read_sql(_MAIN_QUERY_NO_SIXTH_FORM, engine)
|
||||
except Exception as exc2:
|
||||
print(f"Warning: Could not load school data from marts: {exc2}")
|
||||
return pd.DataFrame()
|
||||
else:
|
||||
print(f"Warning: Could not load school data from marts: {exc}")
|
||||
return pd.DataFrame()
|
||||
except Exception as exc:
|
||||
print(f"Warning: Could not load school data from marts: {exc}")
|
||||
return pd.DataFrame()
|
||||
@@ -345,8 +226,6 @@ def load_school_data_as_dataframe() -> pd.DataFrame:
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = translate_gias_code_columns(df)
|
||||
|
||||
# Build address string
|
||||
df["address"] = df.apply(
|
||||
lambda r: ", ".join(
|
||||
@@ -567,6 +446,30 @@ def get_supplementary_data(db: Session, urn: int) -> dict:
|
||||
else None
|
||||
)
|
||||
|
||||
# Parent View
|
||||
pv = safe_query(FactParentView, "urn")
|
||||
result["parent_view"] = (
|
||||
{
|
||||
"survey_date": pv.survey_date.isoformat() if pv.survey_date else None,
|
||||
"total_responses": pv.total_responses,
|
||||
"q_happy_pct": pv.q_happy_pct,
|
||||
"q_safe_pct": pv.q_safe_pct,
|
||||
"q_behaviour_pct": pv.q_behaviour_pct,
|
||||
"q_bullying_pct": pv.q_bullying_pct,
|
||||
"q_communication_pct": pv.q_communication_pct,
|
||||
"q_progress_pct": pv.q_progress_pct,
|
||||
"q_teaching_pct": pv.q_teaching_pct,
|
||||
"q_information_pct": pv.q_information_pct,
|
||||
"q_curriculum_pct": pv.q_curriculum_pct,
|
||||
"q_future_pct": pv.q_future_pct,
|
||||
"q_leadership_pct": pv.q_leadership_pct,
|
||||
"q_wellbeing_pct": pv.q_wellbeing_pct,
|
||||
"q_recommend_pct": pv.q_recommend_pct,
|
||||
}
|
||||
if pv
|
||||
else None
|
||||
)
|
||||
|
||||
# Census (latest year of fact_pupil_characteristics)
|
||||
pc = safe_query(FactPupilCharacteristics, "urn", "year")
|
||||
result["census"] = (
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
"""GIAS code -> name dictionaries.
|
||||
|
||||
GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
|
||||
— do not edit by hand; rerun the script when the dbt drift test warns.
|
||||
The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
|
||||
must be byte-identical (enforced by backend/tests/test_gias_codes.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SCHOOL_TYPE: dict[int, str] = {
|
||||
1: "Community school",
|
||||
2: "Voluntary aided school",
|
||||
3: "Voluntary controlled school",
|
||||
5: "Foundation school",
|
||||
6: "City technology college",
|
||||
7: "Community special school",
|
||||
8: "Non-maintained special school",
|
||||
10: "Other independent special school",
|
||||
11: "Other independent school",
|
||||
12: "Foundation special school",
|
||||
14: "Pupil referral unit",
|
||||
15: "Local authority nursery school",
|
||||
18: "Further education",
|
||||
24: "Secure units",
|
||||
25: "Offshore schools",
|
||||
26: "Service children's education",
|
||||
27: "Miscellaneous",
|
||||
28: "Academy sponsor led",
|
||||
29: "Higher education institutions",
|
||||
30: "Welsh establishment",
|
||||
31: "Sixth form centres",
|
||||
32: "Special post 16 institution",
|
||||
33: "Academy special sponsor led",
|
||||
34: "Academy converter",
|
||||
35: "Free schools",
|
||||
36: "Free schools special",
|
||||
37: "British schools overseas",
|
||||
38: "Free schools alternative provision",
|
||||
39: "Free schools 16 to 19",
|
||||
40: "University technical college",
|
||||
41: "Studio schools",
|
||||
42: "Academy alternative provision converter",
|
||||
43: "Academy alternative provision sponsor led",
|
||||
44: "Academy special converter",
|
||||
45: "Academy 16-19 converter",
|
||||
46: "Academy 16 to 19 sponsor led",
|
||||
49: "Online provider",
|
||||
56: "Institution funded by other government department",
|
||||
57: "Academy secure 16 to 19",
|
||||
}
|
||||
|
||||
ESTABLISHMENT_STATUS: dict[int, str] = {
|
||||
1: "Open",
|
||||
2: "Closed",
|
||||
3: "Open, but proposed to close",
|
||||
4: "Proposed to open",
|
||||
}
|
||||
|
||||
PHASE_OF_EDUCATION: dict[int, str] = {
|
||||
0: "Not applicable",
|
||||
1: "Nursery",
|
||||
2: "Primary",
|
||||
3: "Middle deemed primary",
|
||||
4: "Secondary",
|
||||
5: "Middle deemed secondary",
|
||||
6: "16 plus",
|
||||
7: "All-through",
|
||||
}
|
||||
|
||||
OFFICIAL_SIXTH_FORM: dict[int, str] = {
|
||||
0: "Not applicable",
|
||||
1: "Has a sixth form",
|
||||
2: "Does not have a sixth form",
|
||||
}
|
||||
|
||||
RELIGIOUS_CHARACTER: dict[int, str] = {
|
||||
0: "Does not apply",
|
||||
2: "Church of England",
|
||||
3: "Roman Catholic",
|
||||
4: "Methodist",
|
||||
5: "Jewish",
|
||||
6: "None",
|
||||
7: "Muslim",
|
||||
8: "Seventh Day Adventist",
|
||||
9: "Church of England/Methodist",
|
||||
10: "Methodist/Church of England",
|
||||
11: "Church of England/Roman Catholic",
|
||||
12: "Church of England/United Reformed Church",
|
||||
13: "Roman Catholic/Church of England",
|
||||
14: "Quaker",
|
||||
15: "Christian",
|
||||
16: "United Reformed Church",
|
||||
17: "Congregational Church",
|
||||
18: "Free Church",
|
||||
19: "Church of England/Free Church",
|
||||
20: "Church of England/Christian",
|
||||
21: "Sikh",
|
||||
22: "Greek Orthodox",
|
||||
24: "Buddhist",
|
||||
25: "Hindu",
|
||||
26: "Moravian",
|
||||
28: "Inter- / non- denominational",
|
||||
29: "Multi-faith",
|
||||
30: "Church of England/Methodist/United Reform Church/Baptist",
|
||||
31: "Anglican",
|
||||
32: "Anglican/Christian",
|
||||
33: "Anglican/Evangelical",
|
||||
34: "Anglican/Church of England",
|
||||
35: "Catholic",
|
||||
36: "Charadi Jewish",
|
||||
37: "Christian/Evangelical",
|
||||
38: "Christian Science",
|
||||
39: "Christian/Methodist",
|
||||
40: "Christian/non-denominational",
|
||||
41: "Church of England/Evangelical",
|
||||
42: "Islam",
|
||||
43: "Orthodox Jewish",
|
||||
44: "Plymouth Brethren Christian Church",
|
||||
45: "Protestant",
|
||||
46: "Protestant/Evangelical",
|
||||
47: "Reformed Baptist",
|
||||
48: "Roman Catholic/Anglican",
|
||||
49: "Sunni Deobandi",
|
||||
}
|
||||
|
||||
ADMISSIONS_POLICY: dict[int, str] = {
|
||||
0: "Not applicable",
|
||||
2: "Selective",
|
||||
4: "Non-selective",
|
||||
}
|
||||
|
||||
|
||||
def translate(code, mapping: dict[int, str]) -> str | None:
|
||||
"""Translate a GIAS code to its display name.
|
||||
|
||||
None/NaN -> None (column absent or suppressed). Unknown codes degrade to
|
||||
"Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
|
||||
"""
|
||||
if code is None or (isinstance(code, float) and math.isnan(code)):
|
||||
return None
|
||||
code = int(code)
|
||||
if code not in mapping:
|
||||
logger.warning("Unknown GIAS code %s (not in dictionary)", code)
|
||||
return f"Unknown ({code})"
|
||||
return mapping[code]
|
||||
@@ -433,25 +433,6 @@ def _apply_schema_alterations():
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _apply_schema_drops():
|
||||
"""
|
||||
Drop tables retired from the schema. Idempotent (DROP … IF EXISTS), so it's
|
||||
safe to run on every migration. Add entries here when a model is removed.
|
||||
"""
|
||||
drops = [
|
||||
# v6: Ofsted Parent View feature removed
|
||||
"DROP TABLE IF EXISTS marts.fact_parent_view CASCADE",
|
||||
]
|
||||
from sqlalchemy import text as sa_text
|
||||
with engine.connect() as conn:
|
||||
for stmt in drops:
|
||||
try:
|
||||
conn.execute(sa_text(stmt))
|
||||
except Exception as e:
|
||||
print(f" Warning: drop skipped ({e})")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def run_full_migration(geocode: bool = False) -> bool:
|
||||
"""
|
||||
Run a complete migration: drop all tables and reimport from CSV.
|
||||
@@ -498,9 +479,6 @@ def run_full_migration(geocode: bool = False) -> bool:
|
||||
print("Applying column additions to supplementary tables...")
|
||||
_apply_schema_alterations()
|
||||
|
||||
print("Dropping retired tables...")
|
||||
_apply_schema_drops()
|
||||
|
||||
print("\nLoading CSV data...")
|
||||
df = load_csv_data(settings.data_dir)
|
||||
|
||||
|
||||
+28
-6
@@ -17,22 +17,21 @@ class DimSchool(Base):
|
||||
|
||||
urn = Column(Integer, primary_key=True)
|
||||
school_name = Column(String(255), nullable=False)
|
||||
phase_code = Column(Integer)
|
||||
school_type_code = Column(Integer)
|
||||
phase = Column(String(100))
|
||||
school_type = Column(String(100))
|
||||
academy_trust_name = Column(String(255))
|
||||
academy_trust_uid = Column(String(20))
|
||||
religious_character_code = Column(Integer)
|
||||
religious_character = Column(String(100))
|
||||
gender = Column(String(20))
|
||||
age_range = Column(String(20))
|
||||
has_sixth_form = Column(Boolean)
|
||||
capacity = Column(Integer)
|
||||
total_pupils = Column(Integer)
|
||||
headteacher_name = Column(String(200))
|
||||
website = Column(String(255))
|
||||
telephone = Column(String(30))
|
||||
status_code = Column(Integer)
|
||||
status = Column(String(50))
|
||||
nursery_provision = Column(Boolean)
|
||||
admissions_policy_code = Column(Integer)
|
||||
admissions_policy = Column(String(50))
|
||||
# Denormalised Ofsted summary (updated by monthly pipeline)
|
||||
ofsted_grade = Column(Integer)
|
||||
ofsted_date = Column(Date)
|
||||
@@ -150,6 +149,29 @@ class FactOfstedInspection(Base):
|
||||
report_url = Column(Text)
|
||||
|
||||
|
||||
class FactParentView(Base):
|
||||
"""Ofsted Parent View survey — latest per school."""
|
||||
__tablename__ = "fact_parent_view"
|
||||
__table_args__ = MARTS
|
||||
|
||||
urn = Column(Integer, primary_key=True)
|
||||
survey_date = Column(Date)
|
||||
total_responses = Column(Integer)
|
||||
q_happy_pct = Column(Float)
|
||||
q_safe_pct = Column(Float)
|
||||
q_behaviour_pct = Column(Float)
|
||||
q_bullying_pct = Column(Float)
|
||||
q_communication_pct = Column(Float)
|
||||
q_progress_pct = Column(Float)
|
||||
q_teaching_pct = Column(Float)
|
||||
q_information_pct = Column(Float)
|
||||
q_curriculum_pct = Column(Float)
|
||||
q_future_pct = Column(Float)
|
||||
q_leadership_pct = Column(Float)
|
||||
q_wellbeing_pct = Column(Float)
|
||||
q_recommend_pct = Column(Float)
|
||||
|
||||
|
||||
class FactAdmissions(Base):
|
||||
"""School admissions — one row per URN per year."""
|
||||
__tablename__ = "fact_admissions"
|
||||
|
||||
@@ -543,8 +543,6 @@ SCHOOL_COLUMNS = [
|
||||
"postcode",
|
||||
"religious_denomination",
|
||||
"age_range",
|
||||
"has_sixth_form",
|
||||
"status",
|
||||
"gender",
|
||||
"admissions_policy",
|
||||
"ofsted_grade",
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
"""Tests for the GIAS code->name dictionaries (spec 2026-07-09).
|
||||
|
||||
The dictionaries are generated from the live GIAS bulk CSV by
|
||||
pipeline/scripts/generate_gias_codes.py — these tests assert the module's
|
||||
contract, key sentinel values the marts/UI depend on, and that the pipeline
|
||||
copy has not drifted from the canonical backend module.
|
||||
"""
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from backend.gias_codes import (
|
||||
ADMISSIONS_POLICY,
|
||||
ESTABLISHMENT_STATUS,
|
||||
OFFICIAL_SIXTH_FORM,
|
||||
PHASE_OF_EDUCATION,
|
||||
RELIGIOUS_CHARACTER,
|
||||
SCHOOL_TYPE,
|
||||
translate,
|
||||
)
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_translate_known_code():
|
||||
open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open")
|
||||
assert translate(open_code, ESTABLISHMENT_STATUS) == "Open"
|
||||
|
||||
|
||||
def test_translate_unknown_code_degrades_gracefully():
|
||||
assert translate(9999, ESTABLISHMENT_STATUS) == "Unknown (9999)"
|
||||
|
||||
|
||||
def test_translate_none_and_nan_return_none():
|
||||
assert translate(None, ESTABLISHMENT_STATUS) is None
|
||||
assert translate(float("nan"), ESTABLISHMENT_STATUS) is None
|
||||
|
||||
|
||||
def test_translate_accepts_float_codes():
|
||||
# pd.read_sql yields float columns when NULLs are present
|
||||
open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open")
|
||||
assert translate(float(open_code), ESTABLISHMENT_STATUS) == "Open"
|
||||
|
||||
|
||||
def test_sentinel_names_present():
|
||||
"""Names the marts/UI compare against must exist verbatim."""
|
||||
assert "Open" in ESTABLISHMENT_STATUS.values()
|
||||
assert "Open, but proposed to close" in ESTABLISHMENT_STATUS.values()
|
||||
assert "Has a sixth form" in OFFICIAL_SIXTH_FORM.values()
|
||||
assert "Primary" in PHASE_OF_EDUCATION.values()
|
||||
assert "Secondary" in PHASE_OF_EDUCATION.values()
|
||||
assert "Does not apply" in RELIGIOUS_CHARACTER.values()
|
||||
assert all(len(d) > 0 for d in (
|
||||
SCHOOL_TYPE, ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION,
|
||||
OFFICIAL_SIXTH_FORM, RELIGIOUS_CHARACTER, ADMISSIONS_POLICY,
|
||||
))
|
||||
|
||||
|
||||
def test_pipeline_copy_is_identical():
|
||||
canonical = (REPO / "backend" / "gias_codes.py").read_text()
|
||||
copy = (REPO / "pipeline" / "scripts" / "gias_codes.py").read_text()
|
||||
assert canonical == copy, (
|
||||
"pipeline/scripts/gias_codes.py has drifted from backend/gias_codes.py — "
|
||||
"regenerate with pipeline/scripts/generate_gias_codes.py and copy the file"
|
||||
)
|
||||
|
||||
|
||||
def test_seed_matches_dictionaries():
|
||||
import csv
|
||||
fields = {
|
||||
"school_type": SCHOOL_TYPE,
|
||||
"establishment_status": ESTABLISHMENT_STATUS,
|
||||
"phase_of_education": PHASE_OF_EDUCATION,
|
||||
"official_sixth_form": OFFICIAL_SIXTH_FORM,
|
||||
"religious_character": RELIGIOUS_CHARACTER,
|
||||
"admissions_policy": ADMISSIONS_POLICY,
|
||||
}
|
||||
seed_path = REPO / "pipeline" / "transform" / "seeds" / "gias_code_names.csv"
|
||||
seed: dict[str, dict[int, str]] = {k: {} for k in fields}
|
||||
with open(seed_path, newline="") as fh:
|
||||
for row in csv.DictReader(fh):
|
||||
seed[row["field"]][int(row["code"])] = row["name"]
|
||||
assert seed == fields
|
||||
@@ -1,132 +0,0 @@
|
||||
"""API-boundary translation: marts now carry GIAS codes; the DataFrame the
|
||||
rest of the backend sees must carry today's name strings."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from backend.data_loader import _missing_column_name, translate_gias_code_columns
|
||||
from backend.gias_codes import ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION
|
||||
|
||||
|
||||
def _code_for(mapping, name):
|
||||
return next(c for c, n in mapping.items() if n == name)
|
||||
|
||||
|
||||
def test_codes_become_todays_names():
|
||||
df = pd.DataFrame([{
|
||||
"urn": 1,
|
||||
"phase_code": float(_code_for(PHASE_OF_EDUCATION, "Primary")),
|
||||
"school_type_code": np.nan,
|
||||
"status_code": float(_code_for(ESTABLISHMENT_STATUS, "Open, but proposed to close")),
|
||||
"religious_character_code": np.nan,
|
||||
"admissions_policy_code": np.nan,
|
||||
}])
|
||||
out = translate_gias_code_columns(df)
|
||||
row = out.iloc[0]
|
||||
assert row["phase"] == "Primary"
|
||||
assert row["status"] == "Open, but proposed to close"
|
||||
assert row["school_type"] is None
|
||||
assert row["religious_denomination"] is None
|
||||
assert row["admissions_policy"] is None
|
||||
|
||||
|
||||
def test_unknown_code_degrades_not_blanks():
|
||||
df = pd.DataFrame([{"urn": 1, "phase_code": 9999.0}])
|
||||
out = translate_gias_code_columns(df)
|
||||
assert out.iloc[0]["phase"] == "Unknown (9999)"
|
||||
|
||||
|
||||
def test_missing_code_columns_are_a_noop():
|
||||
"""Old-schema DataFrames (tests, pre-pipeline DBs) pass through untouched."""
|
||||
df = pd.DataFrame([{"urn": 1, "phase": "Primary", "status": "Open"}])
|
||||
out = translate_gias_code_columns(df)
|
||||
assert out.iloc[0]["phase"] == "Primary"
|
||||
assert out.iloc[0]["status"] == "Open"
|
||||
|
||||
|
||||
def _fake_exc(orig_message):
|
||||
"""A stand-in for sqlalchemy.exc.ProgrammingError: str(exc) embeds the
|
||||
full SQL statement (deliberately containing every column name below, to
|
||||
prove the matcher doesn't fall back to it), while .orig carries the real
|
||||
DBAPI error message naming only the offending column."""
|
||||
exc = Exception(
|
||||
"SELECT s.phase_code, s.school_type_code, s.religious_character_code, "
|
||||
"s.status_code, s.admissions_policy_code, s.has_sixth_form FROM ... "
|
||||
f"[SQL: ...] (Background on this error at: https://...)"
|
||||
)
|
||||
exc.orig = Exception(orig_message) if orig_message is not None else None
|
||||
return exc
|
||||
|
||||
|
||||
def test_missing_column_name_quoted():
|
||||
assert _missing_column_name(_fake_exc('column "phase_code" does not exist')) == "phase_code"
|
||||
|
||||
|
||||
def test_missing_column_name_unquoted():
|
||||
assert _missing_column_name(_fake_exc("column phase_code does not exist")) == "phase_code"
|
||||
|
||||
|
||||
def test_missing_column_name_table_prefixed():
|
||||
assert (
|
||||
_missing_column_name(_fake_exc("column s.has_sixth_form does not exist"))
|
||||
== "has_sixth_form"
|
||||
)
|
||||
|
||||
|
||||
def test_missing_column_name_no_match_returns_none():
|
||||
assert _missing_column_name(_fake_exc("relation \"marts.dim_school\" does not exist")) is None
|
||||
|
||||
|
||||
def test_load_school_data_survives_premigration_marts(monkeypatch):
|
||||
"""Real prod state until the nightly pipeline first rebuilds the mart with
|
||||
the GIAS code columns: marts.dim_school still has the old name columns
|
||||
(phase, school_type, religious_character, status, admissions_policy)
|
||||
instead of the new *_code columns. The first query raises UndefinedColumn
|
||||
on s.phase_code; load_school_data_as_dataframe must retry with the
|
||||
legacy name-column query rather than swallow the error and return (and
|
||||
then have load_school_data cache) an empty DataFrame."""
|
||||
import sqlalchemy.exc
|
||||
from backend import data_loader
|
||||
|
||||
data_loader._df_cache = None
|
||||
data_loader._df_latest_cache = None
|
||||
|
||||
good_df = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"urn": 1,
|
||||
"school_name": "Legacy School",
|
||||
"phase": "Primary",
|
||||
"school_type": "Academy",
|
||||
"status": "Open",
|
||||
}
|
||||
]
|
||||
)
|
||||
calls = []
|
||||
|
||||
def fake_read_sql(query, con):
|
||||
calls.append(query)
|
||||
if len(calls) == 1:
|
||||
raise sqlalchemy.exc.ProgrammingError(
|
||||
statement=str(data_loader._MAIN_QUERY),
|
||||
params=None,
|
||||
orig=Exception(
|
||||
"(psycopg2.errors.UndefinedColumn) column s.phase_code "
|
||||
"does not exist\nLINE 5: s.phase_code,"
|
||||
),
|
||||
)
|
||||
return good_df.copy()
|
||||
|
||||
monkeypatch.setattr(data_loader.pd, "read_sql", fake_read_sql)
|
||||
|
||||
try:
|
||||
df = data_loader.load_school_data_as_dataframe()
|
||||
finally:
|
||||
data_loader._df_cache = None
|
||||
data_loader._df_latest_cache = None
|
||||
|
||||
assert len(calls) == 2, "must retry with the legacy name-column query variant"
|
||||
assert calls[1] is data_loader._MAIN_QUERY_LEGACY_NAMES
|
||||
assert not df.empty
|
||||
assert df["phase"].iloc[0] == "Primary"
|
||||
assert df["status"].iloc[0] == "Open"
|
||||
@@ -1,71 +0,0 @@
|
||||
"""Regression tests for GET /api/schools/{urn}.
|
||||
|
||||
Schools with no performance rows (special post-16 institutions, sixth-form
|
||||
centres, PRUs, brand-new schools) come back from the marts LEFT JOIN with
|
||||
NaN in every numeric column. The endpoint must still serialize them — a NaN
|
||||
that reaches Starlette's JSONResponse raises ValueError (allow_nan=False)
|
||||
and the route 500s, which the frontend then renders as a 404.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def _no_results_school_df() -> pd.DataFrame:
|
||||
"""One school row as produced by the marts query for a school with no
|
||||
performance data: GIAS/location fields partly populated, every
|
||||
results-linked column NaN (including year)."""
|
||||
return pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"urn": 150275,
|
||||
"school_name": "West London Performing Arts Academy",
|
||||
"phase": "Secondary",
|
||||
"school_type": "Special post 16 institution",
|
||||
"trust_name": None,
|
||||
"religious_denomination": "Does not apply",
|
||||
"gender": None,
|
||||
"age_range": "16-25",
|
||||
"admissions_policy": None,
|
||||
"capacity": np.nan,
|
||||
"gias_total_pupils": np.nan,
|
||||
"headteacher_name": None,
|
||||
"website": None,
|
||||
"ofsted_grade": np.nan,
|
||||
"local_authority": "Ealing",
|
||||
"address": "268 Northfield Avenue, London, W5 4UB",
|
||||
"postcode": "W5 4UB",
|
||||
"latitude": 51.4986,
|
||||
"longitude": -0.3148,
|
||||
"year": np.nan,
|
||||
"total_pupils": np.nan,
|
||||
"eligible_pupils": np.nan,
|
||||
"rwm_expected_pct": np.nan,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(monkeypatch):
|
||||
from backend import app as app_module
|
||||
|
||||
monkeypatch.setattr(app_module, "load_school_data", _no_results_school_df)
|
||||
monkeypatch.setattr(
|
||||
app_module, "get_supplementary_data", lambda db, urn: {}
|
||||
)
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def test_school_without_performance_rows_returns_200(client):
|
||||
resp = client.get("/api/schools/150275")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
def test_nan_gias_fields_serialize_as_null(client):
|
||||
info = client.get("/api/schools/150275").json()["school_info"]
|
||||
assert info["capacity"] is None
|
||||
assert info["total_pupils"] is None
|
||||
assert info["school_name"] == "West London Performing Arts Academy"
|
||||
@@ -1,70 +0,0 @@
|
||||
"""Tests for GIAS establishment status exposure.
|
||||
|
||||
"Open, but proposed to close" schools are now kept by the dims; the API must
|
||||
surface `status` on list items and school_info so the UI can render the
|
||||
proposed-to-close marker (listing tag) and notice strip (detail page).
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
PROPOSED = "Open, but proposed to close"
|
||||
|
||||
|
||||
def _schools_df() -> pd.DataFrame:
|
||||
base = {
|
||||
"local_authority": "Testshire",
|
||||
"school_type": "Academy",
|
||||
"phase": "Secondary",
|
||||
"address": "1 Test Street",
|
||||
"town": "Testtown",
|
||||
"postcode": "TS1 1AA",
|
||||
"religious_denomination": None,
|
||||
"gender": "Mixed",
|
||||
"age_range": "11-16",
|
||||
"admissions_policy": None,
|
||||
"has_sixth_form": False,
|
||||
"ofsted_grade": np.nan,
|
||||
"ofsted_date": None,
|
||||
"ofsted_framework": None,
|
||||
"latitude": 51.5,
|
||||
"longitude": -0.1,
|
||||
"year": 202425,
|
||||
"total_pupils": 800,
|
||||
"rwm_expected_pct": np.nan,
|
||||
"attainment_8_score": 48.0,
|
||||
}
|
||||
return pd.DataFrame(
|
||||
[
|
||||
{**base, "urn": 200001, "school_name": "Alpha Academy",
|
||||
"status": "Open"},
|
||||
{**base, "urn": 200002, "school_name": "Sarson High School",
|
||||
"status": PROPOSED},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(monkeypatch):
|
||||
from backend import app as app_module
|
||||
|
||||
monkeypatch.setattr(app_module, "load_latest_school_data", _schools_df)
|
||||
monkeypatch.setattr(app_module, "load_school_data", _schools_df)
|
||||
monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def test_list_payload_includes_status(client):
|
||||
resp = client.get("/api/schools")
|
||||
assert resp.status_code == 200, resp.text
|
||||
by_urn = {s["urn"]: s for s in resp.json()["schools"]}
|
||||
assert by_urn[200001]["status"] == "Open"
|
||||
assert by_urn[200002]["status"] == PROPOSED
|
||||
|
||||
|
||||
def test_detail_payload_includes_status(client):
|
||||
resp = client.get("/api/schools/200002")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["school_info"]["status"] == PROPOSED
|
||||
@@ -1,177 +0,0 @@
|
||||
"""Tests for the GIAS-driven has_sixth_form flag (spec 2026-07-07 §3).
|
||||
|
||||
The filter and payloads must use dim_school.has_sixth_form, not the old
|
||||
age_range-contains-"18" substring heuristic. The key regression case is a
|
||||
16-19 sixth-form college: flag true, but "16-19" contains no "18".
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def _schools_df() -> pd.DataFrame:
|
||||
"""Latest-year snapshot rows as produced by load_latest_school_data."""
|
||||
base = {
|
||||
"local_authority": "Testshire",
|
||||
"school_type": "Academy",
|
||||
"phase": "Secondary",
|
||||
"address": "1 Test Street",
|
||||
"town": "Testtown",
|
||||
"postcode": "TS1 1AA",
|
||||
"religious_denomination": None,
|
||||
"gender": "Mixed",
|
||||
"admissions_policy": None,
|
||||
"ofsted_grade": np.nan,
|
||||
"ofsted_date": None,
|
||||
"ofsted_framework": None,
|
||||
"latitude": 51.5,
|
||||
"longitude": -0.1,
|
||||
"year": 202425,
|
||||
"total_pupils": 1000,
|
||||
"rwm_expected_pct": np.nan,
|
||||
"attainment_8_score": 50.0,
|
||||
}
|
||||
return pd.DataFrame(
|
||||
[
|
||||
# 11-18 school WITH a registered sixth form
|
||||
{**base, "urn": 100001, "school_name": "Alpha High",
|
||||
"age_range": "11-18", "has_sixth_form": True},
|
||||
# 16-19 college: old heuristic said NO ("16-19" has no "18"),
|
||||
# GIAS flag says YES — must appear in the yes-filter results
|
||||
{**base, "urn": 100002, "school_name": "Beta Sixth Form College",
|
||||
"age_range": "16-19", "has_sixth_form": True},
|
||||
# 11-18 age range on paper but NO registered sixth form:
|
||||
# old heuristic said YES, GIAS flag says NO
|
||||
{**base, "urn": 100003, "school_name": "Gamma Academy",
|
||||
"age_range": "11-18", "has_sixth_form": False},
|
||||
# Missing flag (pipeline not yet re-run) — must not crash,
|
||||
# must not match the yes-filter
|
||||
{**base, "urn": 100004, "school_name": "Delta School",
|
||||
"age_range": "11-16", "has_sixth_form": None},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(monkeypatch):
|
||||
from backend import app as app_module
|
||||
|
||||
monkeypatch.setattr(app_module, "load_latest_school_data", _schools_df)
|
||||
monkeypatch.setattr(app_module, "load_school_data", _schools_df)
|
||||
monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def _urns(resp):
|
||||
return sorted(s["urn"] for s in resp.json()["schools"])
|
||||
|
||||
|
||||
def test_filter_yes_uses_flag_not_age_range(client):
|
||||
resp = client.get("/api/schools?has_sixth_form=yes")
|
||||
assert resp.status_code == 200, resp.text
|
||||
# 16-19 college included; 11-18-without-sixth-form excluded
|
||||
assert _urns(resp) == [100001, 100002]
|
||||
|
||||
|
||||
def test_filter_no_uses_flag_not_age_range(client):
|
||||
resp = client.get("/api/schools?has_sixth_form=no")
|
||||
assert resp.status_code == 200, resp.text
|
||||
# Gamma (flag false) and Delta (flag missing => not true)
|
||||
assert _urns(resp) == [100003, 100004]
|
||||
|
||||
|
||||
def test_list_payload_includes_flag(client):
|
||||
resp = client.get("/api/schools")
|
||||
assert resp.status_code == 200, resp.text
|
||||
by_urn = {s["urn"]: s for s in resp.json()["schools"]}
|
||||
assert by_urn[100002]["has_sixth_form"] is True
|
||||
assert by_urn[100003]["has_sixth_form"] is False
|
||||
assert by_urn[100004]["has_sixth_form"] is None
|
||||
|
||||
|
||||
def test_detail_payload_includes_flag(client):
|
||||
resp = client.get("/api/schools/100002")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["school_info"]["has_sixth_form"] is True
|
||||
|
||||
|
||||
def test_detail_payload_serializes_numpy_bool(monkeypatch):
|
||||
"""Once the pipeline has run, has_sixth_form is a real bool dtype column
|
||||
(dbt not_null test guarantees no NULLs), so row access yields
|
||||
numpy.bool_ rather than a Python bool. convert_to_native must handle it —
|
||||
otherwise FastAPI's jsonable_encoder raises ValueError and the detail
|
||||
endpoint 500s (C2)."""
|
||||
from backend import app as app_module
|
||||
|
||||
df = _schools_df()
|
||||
# Drop the row with a None flag — this fixture models the post-pipeline
|
||||
# state where the column is a genuine, fully-populated bool dtype.
|
||||
df = df[df["has_sixth_form"].notna()].reset_index(drop=True)
|
||||
df["has_sixth_form"] = df["has_sixth_form"].astype(bool)
|
||||
assert df["has_sixth_form"].dtype == bool
|
||||
|
||||
monkeypatch.setattr(app_module, "load_school_data", lambda: df)
|
||||
monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
|
||||
client = TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
resp = client.get("/api/schools/100002")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["school_info"]["has_sixth_form"] is True
|
||||
|
||||
|
||||
def test_load_school_data_survives_missing_has_sixth_form_column(monkeypatch):
|
||||
"""Real prod state until the nightly pipeline first rebuilds the mart:
|
||||
marts.dim_school lacks has_sixth_form entirely. The first query raises
|
||||
UndefinedColumn; load_school_data_as_dataframe must retry without the
|
||||
column (synthesizing it as None) rather than swallow the error and
|
||||
return (and then have load_school_data cache) an empty DataFrame (C1)."""
|
||||
import sqlalchemy.exc
|
||||
from backend import data_loader
|
||||
|
||||
data_loader._df_cache = None
|
||||
data_loader._df_latest_cache = None
|
||||
|
||||
good_df = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"urn": 1,
|
||||
"school_name": "Fallback School",
|
||||
"school_type": "Academy",
|
||||
"has_sixth_form": None,
|
||||
}
|
||||
]
|
||||
)
|
||||
calls = []
|
||||
|
||||
def fake_read_sql(query, con):
|
||||
calls.append(query)
|
||||
if len(calls) == 1:
|
||||
# The statement text still contains phase_code, school_type_code,
|
||||
# etc. (it's the full _MAIN_QUERY SELECT list) — that's exactly
|
||||
# the collision this test guards against: matching must be done
|
||||
# against exc.orig (the DBAPI error), not str(exc)/the statement.
|
||||
raise sqlalchemy.exc.ProgrammingError(
|
||||
statement=str(data_loader._MAIN_QUERY),
|
||||
params=None,
|
||||
orig=Exception(
|
||||
"(psycopg2.errors.UndefinedColumn) column s.has_sixth_form "
|
||||
"does not exist"
|
||||
),
|
||||
)
|
||||
return good_df.copy()
|
||||
|
||||
monkeypatch.setattr(data_loader.pd, "read_sql", fake_read_sql)
|
||||
|
||||
try:
|
||||
df = data_loader.load_school_data_as_dataframe()
|
||||
finally:
|
||||
data_loader._df_cache = None
|
||||
data_loader._df_latest_cache = None
|
||||
|
||||
assert len(calls) == 2, "must retry with the no-sixth-form query variant"
|
||||
assert calls[1] is data_loader._MAIN_QUERY_NO_SIXTH_FORM
|
||||
assert not df.empty
|
||||
assert "has_sixth_form" in df.columns
|
||||
assert df["has_sixth_form"].iloc[0] is None
|
||||
@@ -11,8 +11,6 @@ def convert_to_native(value: Any) -> Any:
|
||||
"""Convert numpy types to native Python types for JSON serialization."""
|
||||
if pd.isna(value):
|
||||
return None
|
||||
if isinstance(value, np.bool_):
|
||||
return bool(value)
|
||||
if isinstance(value, (np.integer,)):
|
||||
return int(value)
|
||||
if isinstance(value, (np.floating,)):
|
||||
|
||||
+1
-2
@@ -13,7 +13,7 @@ WHEN TO BUMP:
|
||||
"""
|
||||
|
||||
# Current schema version - increment when models change
|
||||
SCHEMA_VERSION = 6
|
||||
SCHEMA_VERSION = 5
|
||||
|
||||
# Changelog for documentation
|
||||
SCHEMA_CHANGELOG = {
|
||||
@@ -22,5 +22,4 @@ SCHEMA_CHANGELOG = {
|
||||
3: "Added supplementary data tables: ofsted, parent_view, census, admissions, sen_detail, phonics, deprivation, finance; GIAS columns on schools",
|
||||
4: "Added Ofsted Report Card columns to ofsted_inspections (new framework from Nov 2025)",
|
||||
5: "Apply ALTER TABLE additions for RC columns missed by create_all on existing tables",
|
||||
6: "Removed the Ofsted Parent View feature: dropped fact_parent_view table and model",
|
||||
}
|
||||
|
||||
+20
-1
@@ -79,7 +79,8 @@ fail the E2E gate. That's the point: staging absorbs the risk.
|
||||
5. **Bootstrap staging data via Airflow** (no prod dump — staging populates
|
||||
itself from source, exercising the pipeline image end-to-end):
|
||||
- Open the staging Airflow UI (`http://<host>:8081`) and trigger, in order:
|
||||
`school_data_daily`, `school_data_monthly_ofsted`, then the manual-schedule
|
||||
`school_data_daily`, `school_data_monthly_ofsted`,
|
||||
`school_data_monthly_parent_view`, then the manual-schedule
|
||||
`school_data_annual_ees` and `school_data_annual_idaci`.
|
||||
- First runs download from government sources (GIAS, Ofsted, EES, IDACI),
|
||||
run dbt, and sync Typesense — expect the initial backfill to take a while.
|
||||
@@ -129,3 +130,21 @@ token Gitea Actions provides automatically (`secrets.GITEA_TOKEN` — no setup
|
||||
needed), and fails the check only when a finding is rated
|
||||
**severe** (would break prod, leak data, or corrupt data). Minor findings are
|
||||
informational and never block a merge.
|
||||
|
||||
## Comment-triggered fix-ups (@claude)
|
||||
|
||||
Comment `@claude <instruction>` on any PR and `.gitea/workflows/pr-comment.yml`
|
||||
runs `scripts/ci/ai_fixup.py`: it checks out the PR branch, hands the
|
||||
instruction to headless Claude Code (same subscription auth as the reviewer),
|
||||
commits and pushes whatever changed, and replies on the PR with a summary.
|
||||
The push re-runs the PR checks automatically.
|
||||
|
||||
Guard rails:
|
||||
- Only comments from the **repo owner** trigger it (the job pushes code).
|
||||
- Only comments starting with `@claude` — the bot's own replies never re-trigger.
|
||||
- Each comment is one full agentic session on the Claude subscription; batch
|
||||
related asks into one comment rather than several small ones.
|
||||
|
||||
Caveat: if the checks don't re-run after the bot's push, Gitea is suppressing
|
||||
workflows for pushes made with the run token — create a personal access token
|
||||
secret and swap it in for the push, or re-run the checks manually.
|
||||
|
||||
@@ -1,560 +0,0 @@
|
||||
# GIAS OfficialSixthForm Flag Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Ingest GIAS's authoritative `OfficialSixthForm` flag into `marts.dim_school.has_sixth_form` and replace every `age_range contains "18"` heuristic in the backend and frontend with it.
|
||||
|
||||
**Architecture:** Data flows tap → raw → dbt staging → dbt mart → backend SQL → API payload → Next.js components. The GIAS Singer tap must declare the new CSV column (target-postgres only persists declared columns); the dbt staging model renames it; `dim_school` derives a boolean (with a statutory-age fallback for blank GIAS values); the backend exposes it on list + detail payloads and uses it for the `has_sixth_form=yes|no` filter; the frontend badge/note/filter-labels switch from the age-range substring check to the flag.
|
||||
|
||||
**Tech Stack:** Singer SDK (tap), dbt (Postgres), FastAPI + pandas, Next.js + TypeScript, pytest, Jest/RTL.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-07-07-exam-phase-taxonomy-design.md` §3.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- A school **has a sixth form** iff GIAS `OfficialSixthForm (name)` = `"Has a sixth form"`. `"Does not have a sixth form"` and `"Not applicable"` → false. Blank/NULL (rare) → fall back to `statutory_high_age >= 18`.
|
||||
- The public API filter parameter stays `has_sixth_form=yes|no` (unchanged contract).
|
||||
- Filter dropdown labels must drop the age-range parentheticals: "With sixth form" / "Without sixth form" (sixth form ≠ age range).
|
||||
- Never push to `main`; work stays on branch `feat/gias-sixth-form-flag` (create from `docs/exam-phase-taxonomy` so the spec is included, or from `main` if that branch has merged).
|
||||
- The dbt models cannot be run locally (no pipeline DB); dbt changes are verified by review + `python -c` schema asserts + existing CI. Do NOT attempt to start a local server.
|
||||
- The backend marts tables are dbt `table` materializations — rebuilt on every pipeline run, so **no ALTER TABLE migration is needed** for `marts.dim_school`.
|
||||
- Deployment ordering: the tap must run before dbt on the first pipeline run after deploy (this is already the DAG order: extract → transform). Until that run happens, `has_sixth_form` is absent from the DB; the backend must treat a missing column as "flag false / fallback", never crash.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Ingest `OfficialSixthForm (name)` — tap schema + dbt staging
|
||||
|
||||
**Files:**
|
||||
- Modify: `pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py:31-66` (Singer schema)
|
||||
- Modify: `pipeline/transform/models/staging/stg_gias_establishments.sql` (add renamed column)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: raw column `"OfficialSixthForm (name)"` in `raw.gias_establishments`; staging column `official_sixth_form` (text: `Has a sixth form` / `Does not have a sixth form` / `Not applicable` / NULL) consumed by Task 2.
|
||||
|
||||
- [ ] **Step 1: Add the property to the Singer schema**
|
||||
|
||||
In `tap.py`, inside `GIASEstablishmentsStream.schema = th.PropertiesList(...)`, add after the `th.Property("PhaseOfEducation (name)", th.StringType),` line:
|
||||
|
||||
```python
|
||||
th.Property("OfficialSixthForm (name)", th.StringType),
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Verify the tap module still imports and declares the column**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/tudor/projects/school_compare/pipeline/plugins/extractors/tap-uk-gias && \
|
||||
python3 -c "
|
||||
import ast, sys
|
||||
src = open('tap_uk_gias/tap.py').read()
|
||||
ast.parse(src)
|
||||
assert '\"OfficialSixthForm (name)\"' in src.replace(\"'\", '\"')
|
||||
print('OK: tap declares OfficialSixthForm (name)')
|
||||
"
|
||||
```
|
||||
Expected: `OK: tap declares OfficialSixthForm (name)`
|
||||
(Uses `ast.parse` instead of importing because `singer_sdk` is not installed locally.)
|
||||
|
||||
- [ ] **Step 3: Add the column to the staging model**
|
||||
|
||||
In `stg_gias_establishments.sql`, in the `renamed` CTE, add after the `"PhaseOfEducation (name)" as phase,` line:
|
||||
|
||||
```sql
|
||||
nullif(trim("OfficialSixthForm (name)"), '') as official_sixth_form,
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Sanity-check the SQL edit**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
grep -n "official_sixth_form" /Users/tudor/projects/school_compare/pipeline/transform/models/staging/stg_gias_establishments.sql
|
||||
```
|
||||
Expected: one line showing the new column inside the `renamed` CTE (before `from source`).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py pipeline/transform/models/staging/stg_gias_establishments.sql
|
||||
git commit -m "feat(pipeline): ingest GIAS OfficialSixthForm into staging
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Derive `dim_school.has_sixth_form` (dbt mart + schema tests + SQLAlchemy model)
|
||||
|
||||
**Files:**
|
||||
- Modify: `pipeline/transform/models/marts/dim_school.sql` (add derived column)
|
||||
- Modify: `pipeline/transform/models/marts/_marts_schema.yml` (document + test the column)
|
||||
- Modify: `backend/models.py:13-38` (`DimSchool` — add column)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `official_sixth_form` text column from Task 1's staging model.
|
||||
- Produces: `marts.dim_school.has_sixth_form boolean not null`, and `DimSchool.has_sixth_form = Column(Boolean)` for the backend. Task 3 selects it as `s.has_sixth_form`.
|
||||
|
||||
- [ ] **Step 1: Add the derived column to `dim_school.sql`**
|
||||
|
||||
In the `select`, add after the `s.age_range` line (`s.statutory_low_age || '-' || s.statutory_high_age as age_range,`):
|
||||
|
||||
```sql
|
||||
-- Authoritative sixth-form flag (spec §3): GIAS OfficialSixthForm.
|
||||
-- "Not applicable" (nurseries, primaries, PRUs) => false. Blank GIAS
|
||||
-- value (rare, new establishments) falls back to the statutory age range.
|
||||
case
|
||||
when s.official_sixth_form = 'Has a sixth form' then true
|
||||
when s.official_sixth_form in ('Does not have a sixth form', 'Not applicable') then false
|
||||
else coalesce(s.statutory_high_age >= 18, false)
|
||||
end as has_sixth_form,
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add schema documentation + tests in `_marts_schema.yml`**
|
||||
|
||||
Under `- name: dim_school` → `columns:`, add after the `phase` column block:
|
||||
|
||||
```yaml
|
||||
- name: has_sixth_form
|
||||
description: >
|
||||
Authoritative sixth-form flag from GIAS OfficialSixthForm.
|
||||
"Has a sixth form" => true; "Does not have a sixth form" and
|
||||
"Not applicable" => false; blank GIAS value falls back to
|
||||
statutory_high_age >= 18. Replaces the age_range-contains-"18"
|
||||
heuristic (spec 2026-07-07 §3).
|
||||
tests:
|
||||
- not_null
|
||||
- accepted_values:
|
||||
values: [true, false]
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add the column to the `DimSchool` SQLAlchemy model**
|
||||
|
||||
In `backend/models.py`, in `class DimSchool`, add after `age_range = Column(String(20))`:
|
||||
|
||||
```python
|
||||
has_sixth_form = Column(Boolean)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify SQL/YAML/Python all parse**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/tudor/projects/school_compare && \
|
||||
python3 -c "
|
||||
import yaml
|
||||
y = yaml.safe_load(open('pipeline/transform/models/marts/_marts_schema.yml'))
|
||||
dim = [m for m in y['models'] if m['name'] == 'dim_school'][0]
|
||||
cols = [c['name'] for c in dim['columns']]
|
||||
assert 'has_sixth_form' in cols, cols
|
||||
print('OK: schema yml documents has_sixth_form')
|
||||
" && \
|
||||
grep -c "has_sixth_form" pipeline/transform/models/marts/dim_school.sql && \
|
||||
python3 -c "import ast; ast.parse(open('backend/models.py').read()); print('OK: models.py parses')"
|
||||
```
|
||||
Expected: `OK: schema yml documents has_sixth_form`, grep count `>= 1`, `OK: models.py parses`.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add pipeline/transform/models/marts/dim_school.sql pipeline/transform/models/marts/_marts_schema.yml backend/models.py
|
||||
git commit -m "feat(pipeline): derive dim_school.has_sixth_form from GIAS flag
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Backend — expose `has_sixth_form` and replace the filter heuristic
|
||||
|
||||
**Files:**
|
||||
- Modify: `backend/data_loader.py:117-215` (`_MAIN_QUERY` — select the column)
|
||||
- Modify: `backend/schemas.py:536-553` (`SCHOOL_COLUMNS` — include in list payloads)
|
||||
- Modify: `backend/app.py:419-422` (filter) and `backend/app.py:589-610` (detail `school_info`)
|
||||
- Test: `backend/tests/test_sixth_form_flag.py` (new)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `marts.dim_school.has_sixth_form` (Task 2).
|
||||
- Produces: `has_sixth_form: bool | null` field on `GET /api/schools` items and on `GET /api/schools/{urn}` → `school_info`. Filter `GET /api/schools?has_sixth_form=yes|no` now driven by the flag. Frontend (Task 4) reads `school.has_sixth_form`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Create `backend/tests/test_sixth_form_flag.py`:
|
||||
|
||||
```python
|
||||
"""Tests for the GIAS-driven has_sixth_form flag (spec 2026-07-07 §3).
|
||||
|
||||
The filter and payloads must use dim_school.has_sixth_form, not the old
|
||||
age_range-contains-"18" substring heuristic. The key regression case is a
|
||||
16-19 sixth-form college: flag true, but "16-19" contains no "18".
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def _schools_df() -> pd.DataFrame:
|
||||
"""Latest-year snapshot rows as produced by load_latest_school_data."""
|
||||
base = {
|
||||
"local_authority": "Testshire",
|
||||
"school_type": "Academy",
|
||||
"phase": "Secondary",
|
||||
"address": "1 Test Street",
|
||||
"town": "Testtown",
|
||||
"postcode": "TS1 1AA",
|
||||
"religious_denomination": None,
|
||||
"gender": "Mixed",
|
||||
"admissions_policy": None,
|
||||
"ofsted_grade": np.nan,
|
||||
"ofsted_date": None,
|
||||
"ofsted_framework": None,
|
||||
"latitude": 51.5,
|
||||
"longitude": -0.1,
|
||||
"year": 202425,
|
||||
"total_pupils": 1000,
|
||||
"rwm_expected_pct": np.nan,
|
||||
"attainment_8_score": 50.0,
|
||||
}
|
||||
return pd.DataFrame(
|
||||
[
|
||||
# 11-18 school WITH a registered sixth form
|
||||
{**base, "urn": 100001, "school_name": "Alpha High",
|
||||
"age_range": "11-18", "has_sixth_form": True},
|
||||
# 16-19 college: old heuristic said NO ("16-19" has no "18"),
|
||||
# GIAS flag says YES — must appear in the yes-filter results
|
||||
{**base, "urn": 100002, "school_name": "Beta Sixth Form College",
|
||||
"age_range": "16-19", "has_sixth_form": True},
|
||||
# 11-18 age range on paper but NO registered sixth form:
|
||||
# old heuristic said YES, GIAS flag says NO
|
||||
{**base, "urn": 100003, "school_name": "Gamma Academy",
|
||||
"age_range": "11-18", "has_sixth_form": False},
|
||||
# Missing flag (pipeline not yet re-run) — must not crash,
|
||||
# must not match the yes-filter
|
||||
{**base, "urn": 100004, "school_name": "Delta School",
|
||||
"age_range": "11-16", "has_sixth_form": None},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(monkeypatch):
|
||||
from backend import app as app_module
|
||||
|
||||
monkeypatch.setattr(app_module, "load_latest_school_data", _schools_df)
|
||||
monkeypatch.setattr(app_module, "load_school_data", _schools_df)
|
||||
monkeypatch.setattr(app_module, "get_supplementary_data", lambda db, urn: {})
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def _urns(resp):
|
||||
return sorted(s["urn"] for s in resp.json()["schools"])
|
||||
|
||||
|
||||
def test_filter_yes_uses_flag_not_age_range(client):
|
||||
resp = client.get("/api/schools?has_sixth_form=yes")
|
||||
assert resp.status_code == 200, resp.text
|
||||
# 16-19 college included; 11-18-without-sixth-form excluded
|
||||
assert _urns(resp) == [100001, 100002]
|
||||
|
||||
|
||||
def test_filter_no_uses_flag_not_age_range(client):
|
||||
resp = client.get("/api/schools?has_sixth_form=no")
|
||||
assert resp.status_code == 200, resp.text
|
||||
# Gamma (flag false) and Delta (flag missing => not true)
|
||||
assert _urns(resp) == [100003, 100004]
|
||||
|
||||
|
||||
def test_list_payload_includes_flag(client):
|
||||
resp = client.get("/api/schools")
|
||||
assert resp.status_code == 200, resp.text
|
||||
by_urn = {s["urn"]: s for s in resp.json()["schools"]}
|
||||
assert by_urn[100002]["has_sixth_form"] is True
|
||||
assert by_urn[100003]["has_sixth_form"] is False
|
||||
assert by_urn[100004]["has_sixth_form"] is None
|
||||
|
||||
|
||||
def test_detail_payload_includes_flag(client):
|
||||
resp = client.get("/api/schools/100002")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["school_info"]["has_sixth_form"] is True
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && python3 -m pytest backend/tests/test_sixth_form_flag.py -v`
|
||||
Expected: FAIL — `test_filter_yes_uses_flag_not_age_range` asserts `[100001, 100002]` but the age-range heuristic returns `[100001, 100003]`; the payload tests fail with `KeyError: 'has_sixth_form'`.
|
||||
|
||||
- [ ] **Step 3: Select the column in `_MAIN_QUERY`**
|
||||
|
||||
In `backend/data_loader.py`, in `_MAIN_QUERY`, add after `s.age_range,`:
|
||||
|
||||
```sql
|
||||
s.has_sixth_form,
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Include it in list payloads**
|
||||
|
||||
In `backend/schemas.py`, in `SCHOOL_COLUMNS`, add after `"age_range",`:
|
||||
|
||||
```python
|
||||
"has_sixth_form",
|
||||
```
|
||||
|
||||
(`app.py` builds list responses from `SCHOOL_COLUMNS ∩ df.columns`, so a DB that predates the pipeline re-run simply omits the field — no crash.)
|
||||
|
||||
- [ ] **Step 5: Replace the filter heuristic in `app.py`**
|
||||
|
||||
Replace lines 419-422:
|
||||
|
||||
```python
|
||||
if has_sixth_form == "yes":
|
||||
df_latest = df_latest[df_latest["age_range"].str.contains("18", na=False)]
|
||||
elif has_sixth_form == "no":
|
||||
df_latest = df_latest[~df_latest["age_range"].str.contains("18", na=False)]
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```python
|
||||
# GIAS OfficialSixthForm flag (dim_school.has_sixth_form). NULL (flag not
|
||||
# yet populated by the pipeline) is treated as "no sixth form".
|
||||
if has_sixth_form in ("yes", "no"):
|
||||
if "has_sixth_form" in df_latest.columns:
|
||||
flag = df_latest["has_sixth_form"].eq(True)
|
||||
else: # DB predates the pipeline re-run — fall back to age range
|
||||
flag = df_latest["age_range"].str.contains("18", na=False)
|
||||
df_latest = df_latest[flag if has_sixth_form == "yes" else ~flag]
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Add the flag to the detail payload**
|
||||
|
||||
In `backend/app.py` `school_info` dict (line ~598), add after `"age_range": latest.get("age_range", ""),`:
|
||||
|
||||
```python
|
||||
"has_sixth_form": latest.get("has_sixth_form"),
|
||||
```
|
||||
|
||||
(`convert_to_native` already maps NaN/None → null and numpy bools → bool.)
|
||||
|
||||
- [ ] **Step 7: Run the new tests**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && python3 -m pytest backend/tests/test_sixth_form_flag.py -v`
|
||||
Expected: 4 passed.
|
||||
|
||||
- [ ] **Step 8: Run the full backend suite**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && python3 -m pytest backend/tests -v`
|
||||
Expected: all pass (the pre-existing `test_school_details.py` df has no `has_sixth_form` column — `latest.get()` returns None, serialized as null).
|
||||
|
||||
- [ ] **Step 9: Commit**
|
||||
|
||||
```bash
|
||||
git add backend/data_loader.py backend/schemas.py backend/app.py backend/tests/test_sixth_form_flag.py
|
||||
git commit -m "feat(api): drive has_sixth_form filter and payloads from GIAS flag
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Frontend — badge, note, row tag, and filter labels use the flag
|
||||
|
||||
**Files:**
|
||||
- Modify: `nextjs-app/lib/types.ts:10-30` (`School` interface)
|
||||
- Modify: `nextjs-app/components/SecondarySchoolDetailView.tsx:101` (badge + coming-soon note)
|
||||
- Modify: `nextjs-app/components/SecondarySchoolRow.tsx:25-27` (row tag)
|
||||
- Modify: `nextjs-app/components/FilterBar.tsx:370-372` (labels only — param name unchanged)
|
||||
- Test: `nextjs-app/__tests__/components/SecondarySchoolRow.test.tsx` (new)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `has_sixth_form: boolean | null` on both list items and `school_info` (Task 3; both are typed as `School`).
|
||||
- Produces: no new exports — behavior change only.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `nextjs-app/__tests__/components/SecondarySchoolRow.test.tsx`:
|
||||
|
||||
```tsx
|
||||
/**
|
||||
* SecondarySchoolRow — sixth-form tag must come from the GIAS
|
||||
* has_sixth_form flag, not the age_range-contains-"18" heuristic.
|
||||
*/
|
||||
|
||||
import '@testing-library/jest-dom';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { SecondarySchoolRow } from '@/components/SecondarySchoolRow';
|
||||
import type { School } from '@/lib/types';
|
||||
|
||||
const base = {
|
||||
urn: 100002,
|
||||
school_name: 'Beta Sixth Form College',
|
||||
local_authority: 'Testshire',
|
||||
school_type: 'Academy',
|
||||
phase: 'Secondary',
|
||||
gender: 'Mixed',
|
||||
attainment_8_score: 50.0,
|
||||
} as unknown as School;
|
||||
|
||||
describe('SecondarySchoolRow sixth-form tag', () => {
|
||||
it('shows the tag for a 16-19 college with the GIAS flag set', () => {
|
||||
render(
|
||||
<SecondarySchoolRow
|
||||
school={{ ...base, age_range: '16-19', has_sixth_form: true }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText('Sixth form')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the tag for an 11-18 school without a registered sixth form', () => {
|
||||
render(
|
||||
<SecondarySchoolRow
|
||||
school={{ ...base, age_range: '11-18', has_sixth_form: false }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.queryByText('Sixth form')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the tag when the flag is missing (pipeline not yet re-run)', () => {
|
||||
render(
|
||||
<SecondarySchoolRow school={{ ...base, age_range: '11-18' }} />,
|
||||
);
|
||||
expect(screen.queryByText('Sixth form')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run it to verify it fails**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare/nextjs-app && npx jest __tests__/components/SecondarySchoolRow.test.tsx`
|
||||
Expected: FAIL — first test can't find "Sixth form" ("16-19" fails the substring check), second test finds an unexpected "Sixth form" tag. (If TS complains that `has_sixth_form` is not on `School`, that is the same failure — proceed.)
|
||||
|
||||
- [ ] **Step 3: Add the field to the `School` type**
|
||||
|
||||
In `nextjs-app/lib/types.ts`, in `export interface School`, add after `age_range: string | null;`:
|
||||
|
||||
```ts
|
||||
has_sixth_form?: boolean | null;
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Switch `SecondarySchoolRow` to the flag**
|
||||
|
||||
Replace the helper at `SecondarySchoolRow.tsx:25-27`:
|
||||
|
||||
```ts
|
||||
function hasSixthForm(school: School): boolean {
|
||||
return school.age_range?.includes('18') ?? false;
|
||||
}
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```ts
|
||||
function hasSixthForm(school: School): boolean {
|
||||
// GIAS OfficialSixthForm flag; missing (pipeline not yet re-run) => false.
|
||||
return school.has_sixth_form ?? false;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Switch `SecondarySchoolDetailView` to the flag**
|
||||
|
||||
Replace line 101:
|
||||
|
||||
```ts
|
||||
const hasSixthForm = schoolInfo.age_range?.includes('18') ?? false;
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```ts
|
||||
// GIAS OfficialSixthForm flag; missing (pipeline not yet re-run) => false.
|
||||
const hasSixthForm = schoolInfo.has_sixth_form ?? false;
|
||||
```
|
||||
|
||||
(This drives both the header "Sixth form" badge at line ~230 and the "Post-16 destination data coming soon" note at line ~715 — no changes needed there.)
|
||||
|
||||
- [ ] **Step 6: Fix the filter labels in `FilterBar.tsx`**
|
||||
|
||||
Replace:
|
||||
|
||||
```tsx
|
||||
<option value="yes">With sixth form (11-18)</option>
|
||||
<option value="no">Without sixth form (11-16)</option>
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```tsx
|
||||
<option value="yes">With sixth form</option>
|
||||
<option value="no">Without sixth form</option>
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run the new test and verify it passes**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare/nextjs-app && npx jest __tests__/components/SecondarySchoolRow.test.tsx`
|
||||
Expected: 3 passed.
|
||||
|
||||
- [ ] **Step 8: Run the full frontend checks**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare/nextjs-app && npx tsc --noEmit && npx jest`
|
||||
Expected: typecheck clean, all Jest suites pass.
|
||||
|
||||
- [ ] **Step 9: Verify no heuristic remains**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
grep -rn "includes('18')\|contains(\"18\")" /Users/tudor/projects/school_compare/nextjs-app/components /Users/tudor/projects/school_compare/backend --include="*.tsx" --include="*.ts" --include="*.py" | grep -v test
|
||||
```
|
||||
Expected: only the documented fallback inside `app.py` (DB-predates-pipeline branch); no other hits.
|
||||
|
||||
- [ ] **Step 10: Commit**
|
||||
|
||||
```bash
|
||||
git add nextjs-app/lib/types.ts nextjs-app/components/SecondarySchoolRow.tsx nextjs-app/components/SecondarySchoolDetailView.tsx nextjs-app/components/FilterBar.tsx nextjs-app/__tests__/components/SecondarySchoolRow.test.tsx
|
||||
git commit -m "feat(ui): sixth-form badge, note and filter labels use GIAS flag
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Update the spec status + PR
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/superpowers/specs/2026-07-07-exam-phase-taxonomy-design.md` (§3 "Pipeline change (future work)" → implemented)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: everything above merged into the branch.
|
||||
- Produces: PR ready for review; e2e journeys are the promotion gate (no journey currently exercises the sixth-form filter, and the API contract is unchanged, so no e2e change is required — state this in the PR body).
|
||||
|
||||
- [ ] **Step 1: Mark spec §3 pipeline change as implemented**
|
||||
|
||||
In the spec, change the §3 heading `### Pipeline change (future work)` to `### Pipeline change (implemented 2026-07-07)` and append one line at the end of that subsection:
|
||||
|
||||
```markdown
|
||||
Implemented in `feat/gias-sixth-form-flag` — see
|
||||
`docs/superpowers/plans/2026-07-07-gias-sixth-form-flag.md`.
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add docs/superpowers/specs/2026-07-07-exam-phase-taxonomy-design.md
|
||||
git commit -m "docs: mark sixth-form flag pipeline change implemented
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Push and open the PR (Gitea)**
|
||||
|
||||
Push the branch, then create the PR against `main` using the Gitea API via the git credential helper (token-header auth 401s on this Gitea; basic auth from `git credential fill` works):
|
||||
|
||||
```bash
|
||||
git push -u origin feat/gias-sixth-form-flag
|
||||
```
|
||||
|
||||
PR title: `feat: drive sixth-form separation from GIAS OfficialSixthForm flag`
|
||||
PR body must note: (1) API contract unchanged (`has_sixth_form=yes|no`), (2) flag is NULL until the next pipeline run — backend and frontend degrade to "no sixth form" / age-range fallback, (3) no e2e journey change needed, and end with the standard generation footer.
|
||||
|
||||
- [ ] **Step 4: Verify CI passes**
|
||||
|
||||
Watch the PR checks (typecheck, tests, builds, AI review). All must pass before merge; merging deploys to staging automatically.
|
||||
@@ -1,799 +0,0 @@
|
||||
# GIAS Code Dictionaries Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Store the six GIAS classification fields as official DfE integer codes in the marts and translate code → name in application code, leaving the API contract (name strings) unchanged.
|
||||
|
||||
**Architecture:** A generation script downloads the public GIAS bulk CSV and emits the dictionaries (Python dicts + a dbt seed) from real data. The tap ingests the `(code)` columns, staging casts them, `dim_school`/`dim_location` keep only codes, and translation happens in exactly two places: `backend/data_loader.py` right after `pd.read_sql`, and `pipeline/scripts/sync_typesense.py` before indexing. A dbt seed test warns when DfE adds/renames a value; a parity test keeps the backend and pipeline dictionary copies identical.
|
||||
|
||||
**Tech Stack:** Singer SDK tap, dbt (Postgres), FastAPI + pandas, Typesense sync script, pytest.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-07-09-gias-code-dictionaries-design.md`
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Numeric code values are never assumed.** Every literal code used in SQL or yml (status filter, sixth-form derivation, phase cascade) must be verified against `pipeline/transform/seeds/gias_code_names.csv` generated in Task 1 from the live CSV. The literals written in this plan are best-current-knowledge and each carries a verification step.
|
||||
- **Names served by the API must stay byte-identical** to today's strings (e.g. `Does not apply`, `Open, but proposed to close`) — UI heuristics compare exact strings.
|
||||
- The `(name)` columns stay declared in the tap and present in raw; staging stops exposing them.
|
||||
- `dim_school` and `dim_location` status filters must stay identical (API inner-joins them).
|
||||
- Backend tests run via: `uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests -v` (no local pytest exists).
|
||||
- dbt cannot run locally — dbt changes are verified statically (grep / yaml parse) + CI.
|
||||
- Never push to `main`. Work on branch `feat/gias-code-dictionaries` (branch off `docs/gias-code-dictionaries` so the spec is included, or off `main` if that has merged).
|
||||
- Commits end with: `Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>`
|
||||
- Deploy runbook (accepted window, spec §7): merge → deploy → trigger `school_data_daily` immediately. No code-level fallback for the old-schema window.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Dictionary generation script, canonical module, pipeline copy, seed
|
||||
|
||||
**Files:**
|
||||
- Create: `pipeline/scripts/generate_gias_codes.py`
|
||||
- Create: `backend/gias_codes.py` (content generated by the script)
|
||||
- Create: `pipeline/scripts/gias_codes.py` (byte-identical copy)
|
||||
- Create: `pipeline/transform/seeds/gias_code_names.csv` (generated)
|
||||
- Test: `backend/tests/test_gias_codes.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `backend/gias_codes.py` exporting `SCHOOL_TYPE`, `ESTABLISHMENT_STATUS`, `PHASE_OF_EDUCATION`, `OFFICIAL_SIXTH_FORM`, `RELIGIOUS_CHARACTER`, `ADMISSIONS_POLICY` (each `dict[int, str]`) and `translate(code, mapping) -> str | None`. Task 4 imports these; Task 5 imports the pipeline copy; Task 3 reads code literals from the seed CSV.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Create `backend/tests/test_gias_codes.py`:
|
||||
|
||||
```python
|
||||
"""Tests for the GIAS code->name dictionaries (spec 2026-07-09).
|
||||
|
||||
The dictionaries are generated from the live GIAS bulk CSV by
|
||||
pipeline/scripts/generate_gias_codes.py — these tests assert the module's
|
||||
contract, key sentinel values the marts/UI depend on, and that the pipeline
|
||||
copy has not drifted from the canonical backend module.
|
||||
"""
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from backend.gias_codes import (
|
||||
ADMISSIONS_POLICY,
|
||||
ESTABLISHMENT_STATUS,
|
||||
OFFICIAL_SIXTH_FORM,
|
||||
PHASE_OF_EDUCATION,
|
||||
RELIGIOUS_CHARACTER,
|
||||
SCHOOL_TYPE,
|
||||
translate,
|
||||
)
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_translate_known_code():
|
||||
open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open")
|
||||
assert translate(open_code, ESTABLISHMENT_STATUS) == "Open"
|
||||
|
||||
|
||||
def test_translate_unknown_code_degrades_gracefully():
|
||||
assert translate(9999, ESTABLISHMENT_STATUS) == "Unknown (9999)"
|
||||
|
||||
|
||||
def test_translate_none_and_nan_return_none():
|
||||
assert translate(None, ESTABLISHMENT_STATUS) is None
|
||||
assert translate(float("nan"), ESTABLISHMENT_STATUS) is None
|
||||
|
||||
|
||||
def test_translate_accepts_float_codes():
|
||||
# pd.read_sql yields float columns when NULLs are present
|
||||
open_code = next(c for c, n in ESTABLISHMENT_STATUS.items() if n == "Open")
|
||||
assert translate(float(open_code), ESTABLISHMENT_STATUS) == "Open"
|
||||
|
||||
|
||||
def test_sentinel_names_present():
|
||||
"""Names the marts/UI compare against must exist verbatim."""
|
||||
assert "Open" in ESTABLISHMENT_STATUS.values()
|
||||
assert "Open, but proposed to close" in ESTABLISHMENT_STATUS.values()
|
||||
assert "Has a sixth form" in OFFICIAL_SIXTH_FORM.values()
|
||||
assert "Primary" in PHASE_OF_EDUCATION.values()
|
||||
assert "Secondary" in PHASE_OF_EDUCATION.values()
|
||||
assert "Does not apply" in RELIGIOUS_CHARACTER.values()
|
||||
assert all(len(d) > 0 for d in (
|
||||
SCHOOL_TYPE, ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION,
|
||||
OFFICIAL_SIXTH_FORM, RELIGIOUS_CHARACTER, ADMISSIONS_POLICY,
|
||||
))
|
||||
|
||||
|
||||
def test_pipeline_copy_is_identical():
|
||||
canonical = (REPO / "backend" / "gias_codes.py").read_text()
|
||||
copy = (REPO / "pipeline" / "scripts" / "gias_codes.py").read_text()
|
||||
assert canonical == copy, (
|
||||
"pipeline/scripts/gias_codes.py has drifted from backend/gias_codes.py — "
|
||||
"regenerate with pipeline/scripts/generate_gias_codes.py and copy the file"
|
||||
)
|
||||
|
||||
|
||||
def test_seed_matches_dictionaries():
|
||||
import csv
|
||||
fields = {
|
||||
"school_type": SCHOOL_TYPE,
|
||||
"establishment_status": ESTABLISHMENT_STATUS,
|
||||
"phase_of_education": PHASE_OF_EDUCATION,
|
||||
"official_sixth_form": OFFICIAL_SIXTH_FORM,
|
||||
"religious_character": RELIGIOUS_CHARACTER,
|
||||
"admissions_policy": ADMISSIONS_POLICY,
|
||||
}
|
||||
seed_path = REPO / "pipeline" / "transform" / "seeds" / "gias_code_names.csv"
|
||||
seed: dict[str, dict[int, str]] = {k: {} for k in fields}
|
||||
with open(seed_path, newline="") as fh:
|
||||
for row in csv.DictReader(fh):
|
||||
seed[row["field"]][int(row["code"])] = row["name"]
|
||||
assert seed == fields
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests/test_gias_codes.py -v`
|
||||
Expected: FAIL at import — `ModuleNotFoundError: No module named 'backend.gias_codes'`.
|
||||
|
||||
- [ ] **Step 3: Write the generation script**
|
||||
|
||||
Create `pipeline/scripts/generate_gias_codes.py`:
|
||||
|
||||
```python
|
||||
"""Generate GIAS code->name dictionaries from the live bulk CSV.
|
||||
|
||||
Writes:
|
||||
- backend/gias_codes.py (canonical Python module)
|
||||
- pipeline/scripts/gias_codes.py (byte-identical copy)
|
||||
- pipeline/transform/seeds/gias_code_names.csv (dbt seed for drift test)
|
||||
|
||||
Run from the repo root whenever the dbt drift test warns that DfE
|
||||
added/renamed a value: python pipeline/scripts/generate_gias_codes.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
GIAS_URL = (
|
||||
"https://ea-edubase-api-prod.azurewebsites.net"
|
||||
"/edubase/downloads/public/edubasealldata{date}.csv"
|
||||
)
|
||||
|
||||
# (CSV code column, CSV name column, python dict name, seed field key)
|
||||
FIELDS = [
|
||||
("TypeOfEstablishment (code)", "TypeOfEstablishment (name)", "SCHOOL_TYPE", "school_type"),
|
||||
("EstablishmentStatus (code)", "EstablishmentStatus (name)", "ESTABLISHMENT_STATUS", "establishment_status"),
|
||||
("PhaseOfEducation (code)", "PhaseOfEducation (name)", "PHASE_OF_EDUCATION", "phase_of_education"),
|
||||
("OfficialSixthForm (code)", "OfficialSixthForm (name)", "OFFICIAL_SIXTH_FORM", "official_sixth_form"),
|
||||
("ReligiousCharacter (code)", "ReligiousCharacter (name)", "RELIGIOUS_CHARACTER", "religious_character"),
|
||||
("AdmissionsPolicy (code)", "AdmissionsPolicy (name)", "ADMISSIONS_POLICY", "admissions_policy"),
|
||||
]
|
||||
|
||||
MODULE_HEADER = '''"""GIAS code -> name dictionaries.
|
||||
|
||||
GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
|
||||
— do not edit by hand; rerun the script when the dbt drift test warns.
|
||||
The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
|
||||
must be byte-identical (enforced by backend/tests/test_gias_codes.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
'''
|
||||
|
||||
MODULE_FOOTER = '''
|
||||
|
||||
def translate(code, mapping: dict[int, str]) -> str | None:
|
||||
"""Translate a GIAS code to its display name.
|
||||
|
||||
None/NaN -> None (column absent or suppressed). Unknown codes degrade to
|
||||
"Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
|
||||
"""
|
||||
if code is None or (isinstance(code, float) and math.isnan(code)):
|
||||
return None
|
||||
code = int(code)
|
||||
if code not in mapping:
|
||||
logger.warning("Unknown GIAS code %s (not in dictionary)", code)
|
||||
return f"Unknown ({code})"
|
||||
return mapping[code]
|
||||
'''
|
||||
|
||||
|
||||
def download_csv() -> pd.DataFrame:
|
||||
for day in (date.today(), date.today() - timedelta(days=1)):
|
||||
url = GIAS_URL.format(date=day.strftime("%Y%m%d"))
|
||||
print(f"Downloading {url}")
|
||||
resp = requests.get(url, timeout=300)
|
||||
if resp.status_code == 404:
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return pd.read_csv(
|
||||
io.StringIO(resp.content.decode("latin-1")),
|
||||
dtype=str, keep_default_na=False,
|
||||
)
|
||||
sys.exit("GIAS CSV not available for today or yesterday")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
repo = Path(__file__).resolve().parents[2]
|
||||
df = download_csv()
|
||||
|
||||
module_parts = [MODULE_HEADER]
|
||||
seed_rows: list[tuple[str, int, str]] = []
|
||||
|
||||
for code_col, name_col, dict_name, field_key in FIELDS:
|
||||
pairs = (
|
||||
df[[code_col, name_col]]
|
||||
.loc[lambda d: (d[code_col] != "") & (d[name_col] != "")]
|
||||
.drop_duplicates()
|
||||
)
|
||||
mapping = sorted((int(c), n) for c, n in pairs.itertuples(index=False))
|
||||
dupes = len(mapping) - len({c for c, _ in mapping})
|
||||
if dupes:
|
||||
sys.exit(f"{code_col}: {dupes} codes map to multiple names — investigate before generating")
|
||||
lines = [f"{dict_name}: dict[int, str] = {{"]
|
||||
for code, name in mapping:
|
||||
escaped = name.replace('"', '\\"')
|
||||
lines.append(f' {code}: "{escaped}",')
|
||||
lines.append("}\n")
|
||||
module_parts.append("\n".join(lines))
|
||||
seed_rows += [(field_key, code, name) for code, name in mapping]
|
||||
|
||||
module = "\n".join(module_parts) + MODULE_FOOTER
|
||||
|
||||
(repo / "backend" / "gias_codes.py").write_text(module)
|
||||
(repo / "pipeline" / "scripts" / "gias_codes.py").write_text(module)
|
||||
|
||||
seed_path = repo / "pipeline" / "transform" / "seeds" / "gias_code_names.csv"
|
||||
with open(seed_path, "w", newline="") as fh:
|
||||
import csv
|
||||
w = csv.writer(fh)
|
||||
w.writerow(["field", "code", "name"])
|
||||
w.writerows(seed_rows)
|
||||
|
||||
print(f"Wrote backend/gias_codes.py, pipeline/scripts/gias_codes.py, {seed_path.name}")
|
||||
print("\nKey codes for the dbt work (Task 3):")
|
||||
for field in ("establishment_status", "phase_of_education", "official_sixth_form"):
|
||||
print(f" {field}:")
|
||||
for f, code, name in seed_rows:
|
||||
if f == field:
|
||||
print(f" {code} = {name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the generator**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && uv run --with pandas --with requests python pipeline/scripts/generate_gias_codes.py`
|
||||
Expected: downloads the CSV (~100MB, may take a minute), writes the three files, and prints the status/phase/sixth-form code tables. **Record the printed code tables — Task 3 needs them.** If the download fails twice, report BLOCKED (no network or GIAS outage) rather than inventing dictionary content.
|
||||
|
||||
- [ ] **Step 5: Run the tests again**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests/test_gias_codes.py -v`
|
||||
Expected: 7 passed. If `test_sentinel_names_present` fails, the GIAS vocabulary differs from expectations — inspect the generated module and report DONE_WITH_CONCERNS naming the differing value; do not edit the generated names.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add pipeline/scripts/generate_gias_codes.py backend/gias_codes.py pipeline/scripts/gias_codes.py pipeline/transform/seeds/gias_code_names.csv backend/tests/test_gias_codes.py
|
||||
git commit -m "feat: GIAS code->name dictionaries generated from live bulk CSV
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Tap ingests the (code) columns; staging exposes codes, drops names
|
||||
|
||||
**Files:**
|
||||
- Modify: `pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py` (Singer schema)
|
||||
- Modify: `pipeline/transform/models/staging/stg_gias_establishments.sql`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: staging columns `school_type_code`, `status_code`, `phase_code`, `official_sixth_form_code`, `religious_character_code`, `admissions_policy_code` (all int) consumed by Task 3. Staging **stops exposing** `school_type`, `status`, `phase`, `official_sixth_form`, `religious_character`, `admissions_policy` (names stay in raw only).
|
||||
|
||||
- [ ] **Step 1: Add the six (code) properties to the Singer schema**
|
||||
|
||||
In `tap.py`, `GIASEstablishmentsStream.schema`, add each `(code)` property directly above its existing `(name)` sibling:
|
||||
|
||||
```python
|
||||
th.Property("TypeOfEstablishment (code)", th.StringType),
|
||||
th.Property("PhaseOfEducation (code)", th.StringType),
|
||||
th.Property("EstablishmentStatus (code)", th.StringType),
|
||||
th.Property("Gender (name)", ...) # existing line — for placement reference only
|
||||
th.Property("ReligiousCharacter (code)", th.StringType),
|
||||
th.Property("AdmissionsPolicy (code)", th.StringType),
|
||||
th.Property("OfficialSixthForm (code)", th.StringType),
|
||||
```
|
||||
|
||||
(The exact insertion order doesn't matter — the schema is a dict — but keep each `(code)` adjacent to its `(name)` for readability. Do NOT remove any `(name)` property.)
|
||||
|
||||
- [ ] **Step 2: Rewrite the six columns in staging**
|
||||
|
||||
In `stg_gias_establishments.sql` `renamed` CTE, replace:
|
||||
|
||||
```sql
|
||||
"TypeOfEstablishment (name)" as school_type,
|
||||
"PhaseOfEducation (name)" as phase,
|
||||
nullif(trim("OfficialSixthForm (name)"), '') as official_sixth_form,
|
||||
"ReligiousCharacter (name)" as religious_character,
|
||||
"AdmissionsPolicy (name)" as admissions_policy,
|
||||
"EstablishmentStatus (name)" as status,
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```sql
|
||||
cast(nullif(trim("TypeOfEstablishment (code)"), '') as integer) as school_type_code,
|
||||
cast(nullif(trim("PhaseOfEducation (code)"), '') as integer) as phase_code,
|
||||
cast(nullif(trim("OfficialSixthForm (code)"), '') as integer) as official_sixth_form_code,
|
||||
cast(nullif(trim("ReligiousCharacter (code)"), '') as integer) as religious_character_code,
|
||||
cast(nullif(trim("AdmissionsPolicy (code)"), '') as integer) as admissions_policy_code,
|
||||
cast(nullif(trim("EstablishmentStatus (code)"), '') as integer) as status_code,
|
||||
```
|
||||
|
||||
(The name lines are scattered through the CTE — replace each in place; the six name aliases must no longer appear in the model.)
|
||||
|
||||
- [ ] **Step 3: Verify statically**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/tudor/projects/school_compare && \
|
||||
python3 -c "import ast; ast.parse(open('pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py').read()); print('tap OK')" && \
|
||||
grep -c "(code)" pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py && \
|
||||
grep -E "as (school_type|status|phase|official_sixth_form|religious_character|admissions_policy)," pipeline/transform/models/staging/stg_gias_establishments.sql; echo "name-alias grep exit=$? (want 1 = none found)"
|
||||
```
|
||||
Expected: `tap OK`, code-column count `6`, and the final grep finds nothing (exit 1).
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py pipeline/transform/models/staging/stg_gias_establishments.sql
|
||||
git commit -m "feat(pipeline): ingest GIAS code columns; staging exposes codes not names
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Marts store codes; dbt tests + drift test
|
||||
|
||||
**Files:**
|
||||
- Modify: `pipeline/transform/models/marts/dim_school.sql`
|
||||
- Modify: `pipeline/transform/models/marts/dim_location.sql`
|
||||
- Modify: `pipeline/transform/models/marts/_marts_schema.yml`
|
||||
- Create: `pipeline/transform/tests/assert_gias_code_names_match_seed.sql`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: staging code columns from Task 2; code literals from `pipeline/transform/seeds/gias_code_names.csv` (Task 1).
|
||||
- Produces: `dim_school` columns `school_type_code`, `status_code`, `phase_code`, `religious_character_code`, `admissions_policy_code` (int) replacing their string columns; `has_sixth_form` unchanged (bool). Task 4's `_MAIN_QUERY` selects these.
|
||||
|
||||
**Before writing SQL: open `pipeline/transform/seeds/gias_code_names.csv` and confirm the literals below.** Best-current-knowledge values (VERIFY EACH):
|
||||
`establishment_status`: 1 = Open, 3 = "Open, but proposed to close" (2 = Closed, 4 = Proposed to open).
|
||||
`phase_of_education`: 0 = Not applicable, 2 = Primary, 4 = Secondary, 7 = All-through.
|
||||
`official_sixth_form`: 1 = Has a sixth form, 2 = Does not have a sixth form, 0 = Not applicable.
|
||||
If any differ, use the seed's values everywhere below and say so in your report.
|
||||
|
||||
- [ ] **Step 1: Rewrite dim_school.sql derivations in code space**
|
||||
|
||||
Replace the phase cascade block (`case ... end as phase,`) with:
|
||||
|
||||
```sql
|
||||
-- Phase in GIAS code space (see seeds/gias_code_names.csv):
|
||||
-- 2 = Primary, 4 = Secondary, 7 = All-through, 0 = Not applicable.
|
||||
case
|
||||
-- 1. Trust GIAS phase when it's a real value (0 = the catch-all "Not Applicable")
|
||||
when s.phase_code is not null and s.phase_code != 0
|
||||
then s.phase_code
|
||||
-- 2. Infer from statutory age range (independent schools still publish these)
|
||||
when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 2
|
||||
when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 4
|
||||
when s.statutory_low_age is not null and s.statutory_high_age is not null
|
||||
and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 7
|
||||
-- 3. Fallback: infer from school name (covers independents with missing ages)
|
||||
when s.school_name ilike '%primary%'
|
||||
or s.school_name ilike '%infant%'
|
||||
or s.school_name ilike '%junior%'
|
||||
or s.school_name ilike '%preparatory%'
|
||||
or s.school_name ilike '% prep school%'
|
||||
or s.school_name ilike '% prep %'
|
||||
then 2
|
||||
when s.school_name ilike '%secondary%'
|
||||
or s.school_name ilike '%high school%'
|
||||
or s.school_name ilike '%grammar%'
|
||||
or s.school_name ilike '%senior school%'
|
||||
or s.school_name ilike '%upper school%'
|
||||
then 4
|
||||
-- 4. Give up — null renders no phase pill
|
||||
else null
|
||||
end as phase_code,
|
||||
```
|
||||
|
||||
Replace `s.school_type,` with `s.school_type_code,`; `s.religious_character,` with `s.religious_character_code,`; `s.admissions_policy,` with `s.admissions_policy_code,`; `s.status,` with `s.status_code,`.
|
||||
|
||||
Replace the has_sixth_form case with:
|
||||
|
||||
```sql
|
||||
-- GIAS OfficialSixthForm in code space: 1 = has, 2 = does not, 0 = N/A.
|
||||
-- Null (rare, new establishments) falls back to the statutory age range.
|
||||
case
|
||||
when s.official_sixth_form_code = 1 then true
|
||||
when s.official_sixth_form_code in (0, 2) then false
|
||||
else coalesce(s.statutory_high_age >= 18, false)
|
||||
end as has_sixth_form,
|
||||
```
|
||||
|
||||
Replace the status filter with:
|
||||
|
||||
```sql
|
||||
-- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when
|
||||
-- GIAS flips to Closed — marts fully rebuild each run).
|
||||
where s.status_code in (1, 3)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Same filter in dim_location.sql**
|
||||
|
||||
Replace its `where s.status in ('Open', 'Open, but proposed to close')` (and the comment above it) with:
|
||||
|
||||
```sql
|
||||
-- Must match dim_school's status filter exactly (the API inner-joins the two).
|
||||
where s.status_code in (1, 3)
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update _marts_schema.yml**
|
||||
|
||||
Under `dim_school` columns: rename `phase` → `phase_code` (keep the warn-severity not_null, reword description to mention codes); replace the `status` accepted_values block with:
|
||||
|
||||
```yaml
|
||||
- name: status_code
|
||||
description: GIAS EstablishmentStatus code (1 = Open, 3 = Open but proposed to close)
|
||||
tests:
|
||||
- accepted_values:
|
||||
values: [1, 3]
|
||||
```
|
||||
|
||||
Add warn-severity accepted_values for the other codes, values copied from the seed (school_type/religious/admissions lists are long — paste the full code list from `gias_code_names.csv` for each):
|
||||
|
||||
```yaml
|
||||
- name: school_type_code
|
||||
tests:
|
||||
- accepted_values:
|
||||
severity: warn
|
||||
values: [<all school_type codes from the seed>]
|
||||
- name: religious_character_code
|
||||
tests:
|
||||
- accepted_values:
|
||||
severity: warn
|
||||
values: [<all religious_character codes from the seed>]
|
||||
- name: admissions_policy_code
|
||||
tests:
|
||||
- accepted_values:
|
||||
severity: warn
|
||||
values: [<all admissions_policy codes from the seed>]
|
||||
```
|
||||
|
||||
(`<...>` here means: paste the actual comma-separated integers from the seed file — the lists exist by the time this task runs. Leaving a literal `<...>` in the yml is a task failure.)
|
||||
|
||||
`has_sixth_form` tests stay unchanged.
|
||||
|
||||
- [ ] **Step 4: Write the drift test**
|
||||
|
||||
Create `pipeline/transform/tests/assert_gias_code_names_match_seed.sql`:
|
||||
|
||||
```sql
|
||||
-- Warn when the live GIAS CSV carries a (code, name) pair we don't have in
|
||||
-- the dictionary seed — i.e. DfE added or renamed a value. Fix by rerunning
|
||||
-- pipeline/scripts/generate_gias_codes.py and committing the regenerated
|
||||
-- dictionaries + seed together.
|
||||
{{ config(severity='warn') }}
|
||||
|
||||
with raw_pairs as (
|
||||
{% for field_key, code_col, name_col in [
|
||||
('school_type', 'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)'),
|
||||
('establishment_status', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)'),
|
||||
('phase_of_education', 'PhaseOfEducation (code)', 'PhaseOfEducation (name)'),
|
||||
('official_sixth_form', 'OfficialSixthForm (code)', 'OfficialSixthForm (name)'),
|
||||
('religious_character', 'ReligiousCharacter (code)', 'ReligiousCharacter (name)'),
|
||||
('admissions_policy', 'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)')
|
||||
] %}
|
||||
select distinct
|
||||
'{{ field_key }}' as field,
|
||||
cast(nullif(trim("{{ code_col }}"), '') as integer) as code,
|
||||
nullif(trim("{{ name_col }}"), '') as name
|
||||
from {{ source('raw', 'gias_establishments') }}
|
||||
where nullif(trim("{{ code_col }}"), '') is not null
|
||||
and nullif(trim("{{ name_col }}"), '') is not null
|
||||
{% if not loop.last %}union all{% endif %}
|
||||
{% endfor %}
|
||||
)
|
||||
|
||||
select r.*
|
||||
from raw_pairs r
|
||||
left join {{ ref('gias_code_names') }} s
|
||||
on s.field = r.field
|
||||
and s.code = r.code
|
||||
and s.name = r.name
|
||||
where s.field is null
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Verify statically**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/tudor/projects/school_compare && \
|
||||
uv run --with pyyaml python -c "import yaml; yaml.safe_load(open('pipeline/transform/models/marts/_marts_schema.yml')); print('yml OK')" && \
|
||||
grep -c "_code" pipeline/transform/models/marts/dim_school.sql && \
|
||||
grep -n "status_code in (1, 3)" pipeline/transform/models/marts/dim_school.sql pipeline/transform/models/marts/dim_location.sql && \
|
||||
grep -rn "s\.status\b\|s\.phase\b\|s\.school_type\b\|s\.religious_character\b\|s\.admissions_policy\b\|official_sixth_form\b" pipeline/transform/models/marts/dim_school.sql | grep -v "_code"; echo "stale-name grep exit=$? (want 1)"
|
||||
```
|
||||
Expected: `yml OK`, both filters matched, and no stale name-column references (final grep exits 1).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add pipeline/transform/models/marts/dim_school.sql pipeline/transform/models/marts/dim_location.sql pipeline/transform/models/marts/_marts_schema.yml pipeline/transform/tests/assert_gias_code_names_match_seed.sql
|
||||
git commit -m "feat(pipeline): dim_school/dim_location store GIAS codes; seed drift test
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Backend translates at the API boundary
|
||||
|
||||
**Files:**
|
||||
- Modify: `backend/models.py` (DimSchool columns)
|
||||
- Modify: `backend/data_loader.py` (`_MAIN_QUERY` + translation)
|
||||
- Test: `backend/tests/test_gias_translation.py` (new)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `backend/gias_codes.py` dictionaries + `translate` (Task 1); mart code columns (Task 3).
|
||||
- Produces: `translate_gias_code_columns(df) -> df` in `backend/data_loader.py`; after `load_school_data_as_dataframe()` the DataFrame carries today's name columns (`phase`, `school_type`, `status`, `religious_denomination`, `admissions_policy`) — every downstream consumer unchanged.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Create `backend/tests/test_gias_translation.py`:
|
||||
|
||||
```python
|
||||
"""API-boundary translation: marts now carry GIAS codes; the DataFrame the
|
||||
rest of the backend sees must carry today's name strings."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from backend.data_loader import translate_gias_code_columns
|
||||
from backend.gias_codes import ESTABLISHMENT_STATUS, PHASE_OF_EDUCATION
|
||||
|
||||
|
||||
def _code_for(mapping, name):
|
||||
return next(c for c, n in mapping.items() if n == name)
|
||||
|
||||
|
||||
def test_codes_become_todays_names():
|
||||
df = pd.DataFrame([{
|
||||
"urn": 1,
|
||||
"phase_code": float(_code_for(PHASE_OF_EDUCATION, "Primary")),
|
||||
"school_type_code": np.nan,
|
||||
"status_code": float(_code_for(ESTABLISHMENT_STATUS, "Open, but proposed to close")),
|
||||
"religious_character_code": np.nan,
|
||||
"admissions_policy_code": np.nan,
|
||||
}])
|
||||
out = translate_gias_code_columns(df)
|
||||
row = out.iloc[0]
|
||||
assert row["phase"] == "Primary"
|
||||
assert row["status"] == "Open, but proposed to close"
|
||||
assert row["school_type"] is None
|
||||
assert row["religious_denomination"] is None
|
||||
assert row["admissions_policy"] is None
|
||||
|
||||
|
||||
def test_unknown_code_degrades_not_blanks():
|
||||
df = pd.DataFrame([{"urn": 1, "phase_code": 9999.0}])
|
||||
out = translate_gias_code_columns(df)
|
||||
assert out.iloc[0]["phase"] == "Unknown (9999)"
|
||||
|
||||
|
||||
def test_missing_code_columns_are_a_noop():
|
||||
"""Old-schema DataFrames (tests, pre-pipeline DBs) pass through untouched."""
|
||||
df = pd.DataFrame([{"urn": 1, "phase": "Primary", "status": "Open"}])
|
||||
out = translate_gias_code_columns(df)
|
||||
assert out.iloc[0]["phase"] == "Primary"
|
||||
assert out.iloc[0]["status"] == "Open"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify failure**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests/test_gias_translation.py -v`
|
||||
Expected: FAIL — `ImportError: cannot import name 'translate_gias_code_columns'`.
|
||||
|
||||
- [ ] **Step 3: Implement translation in data_loader.py**
|
||||
|
||||
Add near the top of `backend/data_loader.py` (after existing imports):
|
||||
|
||||
```python
|
||||
from .gias_codes import (
|
||||
ADMISSIONS_POLICY,
|
||||
ESTABLISHMENT_STATUS,
|
||||
PHASE_OF_EDUCATION,
|
||||
RELIGIOUS_CHARACTER,
|
||||
SCHOOL_TYPE,
|
||||
translate,
|
||||
)
|
||||
|
||||
# mart code column -> (API name column, dictionary)
|
||||
_GIAS_CODE_COLUMNS = {
|
||||
"phase_code": ("phase", PHASE_OF_EDUCATION),
|
||||
"school_type_code": ("school_type", SCHOOL_TYPE),
|
||||
"status_code": ("status", ESTABLISHMENT_STATUS),
|
||||
"religious_character_code": ("religious_denomination", RELIGIOUS_CHARACTER),
|
||||
"admissions_policy_code": ("admissions_policy", ADMISSIONS_POLICY),
|
||||
}
|
||||
|
||||
|
||||
def translate_gias_code_columns(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Map GIAS code columns to today's name columns (API contract).
|
||||
|
||||
Runs immediately after pd.read_sql so every downstream consumer —
|
||||
filters, PHASE_GROUPS, payloads, /api/filters — keeps seeing names.
|
||||
DataFrames without the code columns (old schema, test fixtures) pass
|
||||
through unchanged.
|
||||
"""
|
||||
for code_col, (name_col, mapping) in _GIAS_CODE_COLUMNS.items():
|
||||
if code_col in df.columns:
|
||||
df[name_col] = df[code_col].map(lambda c: translate(c, mapping))
|
||||
return df
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Switch `_MAIN_QUERY` to code columns and call the translation**
|
||||
|
||||
In `_MAIN_QUERY` replace:
|
||||
`s.phase,` → `s.phase_code,` · `s.school_type,` → `s.school_type_code,` · `s.religious_character AS religious_denomination,` → `s.religious_character_code,` · `s.admissions_policy,` → `s.admissions_policy_code,` · `s.status,` → `s.status_code,`
|
||||
|
||||
In `load_school_data_as_dataframe()`, insert the call immediately after the empty-check and **before** the existing `normalize_school_type` line:
|
||||
|
||||
```python
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = translate_gias_code_columns(df)
|
||||
|
||||
# Build address string
|
||||
...
|
||||
# Normalize school type (existing line — now normalises the translated name)
|
||||
df["school_type"] = df["school_type"].apply(normalize_school_type)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Update DimSchool in models.py**
|
||||
|
||||
Replace `phase = Column(String(100))`, `school_type = Column(String(100))`, `religious_character = Column(String(100))`, `admissions_policy = Column(String(50))`, `status = Column(String(50))` with:
|
||||
|
||||
```python
|
||||
phase_code = Column(Integer)
|
||||
school_type_code = Column(Integer)
|
||||
religious_character_code = Column(Integer)
|
||||
admissions_policy_code = Column(Integer)
|
||||
status_code = Column(Integer)
|
||||
```
|
||||
|
||||
Then check nothing else references the removed attributes:
|
||||
```bash
|
||||
grep -rn "\.phase\b\|\.school_type\b\|\.religious_character\b\|\.admissions_policy\b\|\.status\b" backend/*.py | grep -i "dimschool\|DimSchool"
|
||||
```
|
||||
Expected: no hits (the backend reads via `_MAIN_QUERY`, not ORM attributes). If there are hits, update them to the `_code` columns + translation and note it in your report.
|
||||
|
||||
- [ ] **Step 6: Run the new tests and the whole backend suite**
|
||||
|
||||
Run: `cd /Users/tudor/projects/school_compare && uv run --with-requirements requirements.txt --with pytest --with "httpx==0.27.0" python -m pytest backend/tests -v`
|
||||
Expected: all pass — 3 new + all pre-existing (their fixtures carry name columns; translation is a no-op on them).
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add backend/models.py backend/data_loader.py backend/tests/test_gias_translation.py
|
||||
git commit -m "feat(api): translate GIAS codes to names at the query boundary
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Typesense sync translates before indexing
|
||||
|
||||
**Files:**
|
||||
- Modify: `pipeline/scripts/sync_typesense.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `pipeline/scripts/gias_codes.py` (Task 1), mart code columns (Task 3).
|
||||
- Produces: identical Typesense documents to today (facet values are names).
|
||||
|
||||
- [ ] **Step 1: Switch the SELECT and translate**
|
||||
|
||||
In `sync_typesense.py`: add at the top (the DAG runs `python scripts/sync_typesense.py`, so `scripts/` is `sys.path[0]` and a plain import works):
|
||||
|
||||
```python
|
||||
from gias_codes import PHASE_OF_EDUCATION, RELIGIOUS_CHARACTER, SCHOOL_TYPE, translate
|
||||
```
|
||||
|
||||
In the SQL, replace `s.phase,` → `s.phase_code,`, `s.school_type,` → `s.school_type_code,`, `s.religious_character,` → `s.religious_character_code,`.
|
||||
|
||||
In the document builder, replace:
|
||||
|
||||
```python
|
||||
"phase": row["phase"] or "",
|
||||
"school_type": row["school_type"] or "",
|
||||
```
|
||||
with:
|
||||
```python
|
||||
"phase": translate(row["phase_code"], PHASE_OF_EDUCATION) or "",
|
||||
"school_type": translate(row["school_type_code"], SCHOOL_TYPE) or "",
|
||||
```
|
||||
and:
|
||||
```python
|
||||
if row.get("religious_character"):
|
||||
doc["religious_character"] = row["religious_character"]
|
||||
```
|
||||
with:
|
||||
```python
|
||||
religious_character = translate(row.get("religious_character_code"), RELIGIOUS_CHARACTER)
|
||||
if religious_character:
|
||||
doc["religious_character"] = religious_character
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Verify statically**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/tudor/projects/school_compare && \
|
||||
python3 -c "import ast; ast.parse(open('pipeline/scripts/sync_typesense.py').read()); print('sync OK')" && \
|
||||
grep -n "row\[\"phase\"\]\|row\[\"school_type\"\]\|row\[\"religious_character\"\]" pipeline/scripts/sync_typesense.py; echo "stale grep exit=$? (want 1)"
|
||||
```
|
||||
Expected: `sync OK`, no stale name-column row accesses.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add pipeline/scripts/sync_typesense.py
|
||||
git commit -m "feat(pipeline): typesense sync translates GIAS codes before indexing
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Spec status, PR, deploy runbook
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/superpowers/specs/2026-07-09-gias-code-dictionaries-design.md` (status line)
|
||||
|
||||
- [ ] **Step 1: Mark the spec implemented**
|
||||
|
||||
Change `**Status:** Approved design` to `**Status:** Implemented 2026-07-09 — see docs/superpowers/plans/2026-07-09-gias-code-dictionaries.md`.
|
||||
|
||||
- [ ] **Step 2: Commit and push**
|
||||
|
||||
```bash
|
||||
git add docs/superpowers/specs/2026-07-09-gias-code-dictionaries-design.md
|
||||
git commit -m "docs: mark GIAS code dictionaries spec implemented
|
||||
|
||||
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>"
|
||||
git push -u origin feat/gias-code-dictionaries
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Open the PR (Gitea API via git credential fill — token-header auth 401s)**
|
||||
|
||||
Title: `feat: GIAS classification fields stored as codes, translated in code`
|
||||
Body must include: (1) API contract unchanged — names still served, translation at the query boundary; (2) the **deploy runbook: merge → deploy → trigger `school_data_daily` immediately** (accepted empty-API window until the marts rebuild — spec §7); (3) dictionary maintenance loop (dbt drift test warns → rerun `generate_gias_codes.py` → commit regenerated files); (4) no frontend/e2e changes. End with the standard generation footer.
|
||||
|
||||
- [ ] **Step 4: Watch CI**
|
||||
|
||||
All PR checks must pass. Do not merge — merging triggers the deploy window; the human runs the runbook.
|
||||
@@ -1,215 +0,0 @@
|
||||
# Exam Results Taxonomy — Phase Grouping and Sixth-Form Separation
|
||||
|
||||
**Date:** 2026-07-07
|
||||
**Status:** Approved design (taxonomy/analysis only — no implementation in this doc's scope)
|
||||
|
||||
## Purpose
|
||||
|
||||
Classify every exam-result metric SchoolCompare displays today into four phase
|
||||
groups — **Primary**, **Secondary**, **Sixth form**, **Other** — and define an
|
||||
authoritative rule for separating schools that have a sixth form from those
|
||||
that don't. This document is the reference for:
|
||||
|
||||
1. How the UI should group results sections and rankings by phase.
|
||||
2. The future KS5 (A-level) ingestion work — the Sixth form group lists the
|
||||
concrete DfE metrics as placeholders with source columns.
|
||||
3. Replacing the fragile `age_range contains "18"` heuristic with the GIAS
|
||||
`OfficialSixthForm` flag.
|
||||
|
||||
## 1. Grouping principle
|
||||
|
||||
Metrics are grouped by **the key stage of the assessment**, not by the phase
|
||||
of the school displaying them. An all-through school (4–18) shows metrics in
|
||||
all three exam groups; a pure primary shows only the Primary group.
|
||||
|
||||
| Group | Assessments | Key stage | Taken at age | Data status |
|
||||
|---|---|---|---|---|
|
||||
| **Primary** | KS2 SATs (reading, writing TA, maths, GPS, science TA) | KS2 | 10–11 (Year 6) | ✅ Live — `marts.fact_ks2_performance` |
|
||||
| **Secondary** | GCSEs, Attainment 8 / Progress 8, EBacc | KS4 | 15–16 (Year 11) | ✅ Live — `marts.fact_ks4_performance` |
|
||||
| **Sixth form** | A levels, applied general, tech levels | KS5 (16–18) | 17–18 (Year 12–13) | ⏳ Not ingested — placeholders in §4 |
|
||||
| **Other** | Non-exam context displayed alongside results | n/a | n/a | ✅ Live — various marts |
|
||||
|
||||
Not covered (not displayed today, candidates for future "Other"/Primary):
|
||||
EYFS Good Level of Development, Year 1 Phonics check, Year 4 Multiplication
|
||||
Tables Check, KS1 assessments (no longer published at school level by DfE).
|
||||
|
||||
## 2. Metric-by-metric mapping (current site)
|
||||
|
||||
Every key in `backend/schemas.py` `METRIC_DEFINITIONS` — the single source of
|
||||
truth for what the site displays — mapped to its phase group. `category` is
|
||||
the existing schema category; source columns are the DfE names used at
|
||||
ingestion (legacy performance-tables CSV for KS2, EES for KS4).
|
||||
|
||||
### Primary (KS2 SATs)
|
||||
|
||||
| Metric key | Category | DfE source column |
|
||||
|---|---|---|
|
||||
| `rwm_expected_pct` | expected | `PTRWM_EXP` |
|
||||
| `reading_expected_pct` | expected | `PTREAD_EXP` |
|
||||
| `writing_expected_pct` | expected | `PTWRITTA_EXP` |
|
||||
| `maths_expected_pct` | expected | `PTMAT_EXP` |
|
||||
| `gps_expected_pct` | expected | `PTGPS_EXP` |
|
||||
| `science_expected_pct` | expected | `PTSCITA_EXP` |
|
||||
| `rwm_high_pct` | higher | `PTRWM_HIGH` |
|
||||
| `reading_high_pct` | higher | `PTREAD_HIGH` |
|
||||
| `writing_high_pct` | higher | `PTWRITTA_HIGH` |
|
||||
| `maths_high_pct` | higher | `PTMAT_HIGH` |
|
||||
| `gps_high_pct` | higher | `PTGPS_HIGH` |
|
||||
| `reading_progress` | progress | `READPROG` |
|
||||
| `writing_progress` | progress | `WRITPROG` |
|
||||
| `maths_progress` | progress | `MATPROG` |
|
||||
| `reading_avg_score` | average | `READ_AVERAGE` |
|
||||
| `maths_avg_score` | average | `MAT_AVERAGE` |
|
||||
| `gps_avg_score` | average | `GPS_AVERAGE` |
|
||||
| `rwm_expected_boys_pct` | gender | `PTRWM_EXP_B` |
|
||||
| `rwm_expected_girls_pct` | gender | `PTRWM_EXP_G` |
|
||||
| `rwm_high_boys_pct` | gender | `PTRWM_HIGH_B` |
|
||||
| `rwm_high_girls_pct` | gender | `PTRWM_HIGH_G` |
|
||||
| `rwm_expected_disadvantaged_pct` | equity | `PTRWM_EXP_FSM6CLA1A` |
|
||||
| `rwm_expected_non_disadvantaged_pct` | equity | `PTRWM_EXP_NotFSM6CLA1A` |
|
||||
| `disadvantaged_gap` | equity | `DIFFN_RWM_EXP` |
|
||||
| `reading_absence_pct` | absence | `PTREAD_AT` |
|
||||
| `gps_absence_pct` | absence | `PTGPS_AT` |
|
||||
| `maths_absence_pct` | absence | `PTMAT_AT` |
|
||||
| `writing_absence_pct` | absence | `PTWRITTA_AD` |
|
||||
| `science_absence_pct` | absence | `PTSCITA_AD` |
|
||||
| `rwm_expected_3yr_pct` | trends | `PTRWM_EXP_3YR` |
|
||||
| `reading_avg_3yr` | trends | `READ_AVERAGE_3YR` |
|
||||
| `maths_avg_3yr` | trends | `MAT_AVERAGE_3YR` |
|
||||
|
||||
The absence metrics measure absence *from KS2 tests*, so they belong to
|
||||
Primary even though they are not attainment scores. National comparators for
|
||||
this group come from `marts.fact_ks2_national_averages`.
|
||||
|
||||
### Secondary (KS4 / GCSE)
|
||||
|
||||
| Metric key | Category | EES source column |
|
||||
|---|---|---|
|
||||
| `attainment_8_score` | gcse | `attainment8_average` |
|
||||
| `progress_8_score` | gcse | `progress8_average` |
|
||||
| `english_maths_standard_pass_pct` | gcse | `engmath_94_percent` |
|
||||
| `english_maths_strong_pass_pct` | gcse | `engmath_95_percent` |
|
||||
| `ebacc_entry_pct` | gcse | `ebacc_entering_percent` |
|
||||
| `ebacc_standard_pass_pct` | gcse | `ebacc_94_percent` |
|
||||
| `ebacc_strong_pass_pct` | gcse | `ebacc_95_percent` |
|
||||
| `ebacc_avg_score` | gcse | `ebacc_aps_average` |
|
||||
| `gcse_grade_91_pct` | gcse | `gcse_91_percent` |
|
||||
|
||||
Also stored in `marts.fact_ks4_performance` (and `fact_performance`) but not
|
||||
yet in `METRIC_DEFINITIONS` — Secondary group members when surfaced:
|
||||
`progress_8_lower_ci`, `progress_8_upper_ci`, `progress_8_english`,
|
||||
`progress_8_maths`, `progress_8_ebacc`, `progress_8_open`,
|
||||
`prior_attainment_avg` (KS2 baseline of the GCSE cohort), `sen_pct`.
|
||||
|
||||
### Sixth form (KS5)
|
||||
|
||||
No metrics today. The secondary school detail view renders a static note
|
||||
("Post-16 destination data coming soon") when the school has a sixth form.
|
||||
Placeholders for ingestion are specified in §4.
|
||||
|
||||
### Other (non-exam context)
|
||||
|
||||
Displayed alongside results but not tied to any assessment:
|
||||
|
||||
| Metric key / surface | Category | Source |
|
||||
|---|---|---|
|
||||
| `disadvantaged_pct` | context | KS2 CSV `PTFSM6CLA1A` |
|
||||
| `eal_pct` | context | KS2 CSV `PTEALGRP2` |
|
||||
| `sen_support_pct` | context | KS2 CSV `PSENELK` (KS4 fallback `sen_no_ehcp_pupil_percent`) |
|
||||
| `stability_pct` | context | KS2 CSV `PTMOBN` |
|
||||
| Ofsted grades incl. `sixth_form_provision` / `rc_sixth_form` | — | `marts.fact_ofsted_inspection` |
|
||||
| Admissions (offers, oversubscription) | — | `marts.fact_admissions` |
|
||||
| Finance (per-pupil spend, cost shares) | — | `marts.fact_finance` |
|
||||
| Deprivation (IDACI) | — | `marts.fact_deprivation` |
|
||||
| Pupil characteristics (census) | — | `marts.fact_pupil_characteristics` |
|
||||
|
||||
Note: the context metrics are cohort characteristics of the KS2 cohort at
|
||||
source, but they are presented (and should stay presented) as school-level
|
||||
context, so they group as Other, not Primary.
|
||||
|
||||
## 3. Sixth-form separation
|
||||
|
||||
### Definition (authoritative)
|
||||
|
||||
> A school **has a sixth form** iff GIAS `OfficialSixthForm (name)` =
|
||||
> `"Has a sixth form"` for its URN.
|
||||
|
||||
GIAS values are `Has a sixth form`, `Does not have a sixth form`, and
|
||||
`Not applicable` / blank. `Not applicable` (nurseries, primaries, PRUs) maps
|
||||
to **false**. This field is the DfE's registry flag, updated continuously,
|
||||
and is the only source that correctly classifies:
|
||||
|
||||
- 16–19 sixth-form colleges and UTCs (age ranges like `14-19`, `16-19` that
|
||||
the current substring heuristic misclassifies as *no* sixth form);
|
||||
- schools whose statutory age range extends to 18 on paper but which have no
|
||||
registered post-16 provision.
|
||||
|
||||
### Pipeline change (implemented 2026-07-07)
|
||||
|
||||
1. `stg_gias_establishments.sql`: add
|
||||
`"OfficialSixthForm (name)" as official_sixth_form`.
|
||||
2. `dim_school.sql` (+ `models.py` `DimSchool`, `_marts_schema.yml`): add
|
||||
`has_sixth_form boolean` = `official_sixth_form = 'Has a sixth form'`.
|
||||
3. Expose `has_sixth_form` on the school API payloads.
|
||||
|
||||
Implemented in `feat/gias-sixth-form-flag` — see
|
||||
`docs/superpowers/plans/2026-07-07-gias-sixth-form-flag.md`.
|
||||
|
||||
### Current heuristic — audit of `age_range` ~ "18" sites
|
||||
|
||||
All must migrate to the `has_sixth_form` flag once exposed:
|
||||
|
||||
| Site | Current behaviour |
|
||||
|---|---|
|
||||
| `backend/app.py:419-422` | `/api/schools?has_sixth_form=yes\|no` filters on `age_range.str.contains("18")` |
|
||||
| `nextjs-app/components/SecondarySchoolDetailView.tsx:101` | "Sixth form" badge + coming-soon note from `age_range?.includes('18')` |
|
||||
| `nextjs-app/components/FilterBar.tsx:370-372` | Filter labels hard-code "(11-18)" / "(11-16)" — labels should drop the age-range parenthetical since sixth form ≠ age range |
|
||||
|
||||
Fallback rule: if GIAS is blank for a URN (rare; new establishments), fall
|
||||
back to the age-range heuristic and log the URN.
|
||||
|
||||
### UI separation rules
|
||||
|
||||
- **School page**: schools with `has_sixth_form = true` show a Sixth form
|
||||
results section (placeholder until KS5 data lands); schools without never
|
||||
show it. Badge on the header as today, but driven by the flag.
|
||||
- **Search/rankings filter**: "With sixth form" / "Without sixth form" uses
|
||||
the flag; applies to secondary and all-through phases.
|
||||
- **Comparison**: when comparing a with-sixth-form school against one
|
||||
without, the Sixth form group renders "No sixth form" for the latter
|
||||
rather than blank cells, making the structural difference explicit.
|
||||
|
||||
## 4. Sixth form placeholders — future KS5 ingestion spec
|
||||
|
||||
Source: DfE "A level and other 16 to 18 results" (EES, preferred — matches
|
||||
the KS4 EES tap) or legacy performance-tables `england_ks5final.csv`.
|
||||
Column names below are from the legacy KS5 CSV; verify against the EES
|
||||
release chosen at ingestion time.
|
||||
|
||||
| Proposed metric key | Name | Legacy source column | Type |
|
||||
|---|---|---|---|
|
||||
| `alevel_aps_per_entry` | A level average points per entry | `TALLPPE_ALEV_1618` | score |
|
||||
| `alevel_avg_grade` | A level average grade (e.g. B-) | `TALLPPEGRD_ALEV_1618` | grade |
|
||||
| `academic_aps_per_entry` | Academic qualifications APS per entry | `TALLPPE_ACAD_1618` | score |
|
||||
| `applied_general_aps_per_entry` | Applied general APS per entry | `TALLPPE_AGEN_1618` | score |
|
||||
| `tech_level_aps_per_entry` | Tech level APS per entry | `TALLPPE_TLEV_1618` | score |
|
||||
| `english_progress_1618` | English progress (16–18, unfinished GCSE 4+) | `PROGENG_1618` | score |
|
||||
| `maths_progress_1618` | Maths progress (16–18) | `PROGMAT_1618` | score |
|
||||
| `ks5_cohort_size` | Students at end of 16–18 study | `TALLPUP_1618` | count |
|
||||
| `alevel_3plus_aab_pct` | % achieving AAB+ in ≥2 facilitating subjects | `TAAB2FAC_1618` | percentage |
|
||||
| `ks5_retention_pct` | Retention (completed main programme) | study-programme retention measure | percentage |
|
||||
| `ks5_destinations_pct` | Sustained education/employment destination | 16–18 destination measures dataset | percentage |
|
||||
|
||||
Proposed landing shape mirrors KS4: `stg_ees_ks5.sql` →
|
||||
`int_ks5_with_lineage.sql` → `marts.fact_ks5_performance` (one row per URN
|
||||
per year), joined into `fact_performance`, with a `category: "sixth_form"`
|
||||
(or `"alevel"`) block added to `METRIC_DEFINITIONS`.
|
||||
|
||||
## 5. Out of scope
|
||||
|
||||
- Any implementation (pipeline, API, or UI changes) — this is the taxonomy
|
||||
reference; implementation work items are §3 "Pipeline change", the
|
||||
heuristic migration audit, and §4 ingestion, each to be planned separately.
|
||||
- Middle schools (deemed secondary/primary): they follow the assessment-based
|
||||
grouping automatically — no special casing.
|
||||
- Independent schools: no DfE performance data published; unaffected.
|
||||
@@ -1,182 +0,0 @@
|
||||
# GIAS Code Dictionaries — Codes in Marts, Names in Code
|
||||
|
||||
**Date:** 2026-07-09
|
||||
**Status:** Implemented 2026-07-09 — see docs/superpowers/plans/2026-07-09-gias-code-dictionaries.md
|
||||
|
||||
## Goal
|
||||
|
||||
Six GIAS classification fields are stored in the marts as repeated name
|
||||
strings. Replace them with the official DfE integer codes and translate
|
||||
code → name in application code. After this change the marts carry only
|
||||
codes for:
|
||||
|
||||
| GIAS field | Today (marts, string) | After (marts, int) |
|
||||
|---|---|---|
|
||||
| `TypeOfEstablishment (name)` | `dim_school.school_type` | `school_type_code` |
|
||||
| `EstablishmentStatus (name)` | `dim_school.status` | `status_code` |
|
||||
| `PhaseOfEducation (name)` | `dim_school.phase` | `phase_code` |
|
||||
| `OfficialSixthForm (name)` | (already reduced to `has_sixth_form` bool) | `official_sixth_form_code` in staging only; mart keeps the bool |
|
||||
| `ReligiousCharacter (name)` | `dim_school.religious_character` | `religious_character_code` |
|
||||
| `AdmissionsPolicy (name)` | `dim_school.admissions_policy` | `admissions_policy_code` |
|
||||
|
||||
Motivation: smaller marts and stable enum values for filtering. (Honest
|
||||
sizing note: at ~25k open schools the raw performance win is modest; the
|
||||
durable benefits are storage, DfE-governed vocabulary, and filter values
|
||||
that can't drift with GIAS renames.)
|
||||
|
||||
## Decisions (made during brainstorming)
|
||||
|
||||
1. **GIAS native codes**, not custom enums. The GIAS bulk CSV publishes an
|
||||
official `X (code)` column beside every `X (name)` column. We ingest the
|
||||
DfE's own codes; no invented mapping to maintain.
|
||||
2. **Translation lives in the backend at the API boundary.** The API keeps
|
||||
serving today's name strings; the frontend, e2e journeys, and API
|
||||
consumers are untouched.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Tap (Singer schema)
|
||||
|
||||
Add the six `(code)` columns to `GIASEstablishmentsStream.schema` in
|
||||
`pipeline/plugins/extractors/tap-uk-gias/tap_uk_gias/tap.py`:
|
||||
|
||||
```
|
||||
"TypeOfEstablishment (code)", "EstablishmentStatus (code)",
|
||||
"PhaseOfEducation (code)", "OfficialSixthForm (code)",
|
||||
"ReligiousCharacter (code)", "AdmissionsPolicy (code)"
|
||||
```
|
||||
|
||||
The `(name)` columns **stay declared** — raw keeps both so we can detect
|
||||
dictionary drift (§4) and regenerate dictionaries from live data.
|
||||
|
||||
### 2. Staging (`stg_gias_establishments.sql`)
|
||||
|
||||
- Add int casts: `school_type_code`, `status_code`, `phase_code`,
|
||||
`official_sixth_form_code`, `religious_character_code`,
|
||||
`admissions_policy_code` (all `cast(nullif(trim(...), '') as integer)`).
|
||||
- Remove the corresponding name columns from the staging select
|
||||
(`school_type`, `status`, `phase`, `official_sixth_form`,
|
||||
`religious_character`, `admissions_policy`). Names live only in raw.
|
||||
|
||||
### 3. Marts
|
||||
|
||||
**`dim_school`** stores codes only:
|
||||
|
||||
- `school_type_code`, `status_code`, `phase_code`,
|
||||
`religious_character_code`, `admissions_policy_code` replace their
|
||||
string columns.
|
||||
- Status filter becomes `where status_code in (<open>, <proposed-to-close>)`.
|
||||
The numeric values are read from live raw data at implementation time
|
||||
(`select distinct "EstablishmentStatus (code)", "EstablishmentStatus (name)"`),
|
||||
never assumed from memory. Same filter in `dim_location`.
|
||||
- `has_sixth_form` derives from `official_sixth_form_code`
|
||||
(`<has-code>` → true, `<does-not>/<not-applicable>` → false, null →
|
||||
`statutory_high_age >= 18` fallback). The `lower(trim(...))` string guard
|
||||
becomes obsolete and is removed.
|
||||
- `phase_code` derivation keeps today's cascade but emits codes:
|
||||
1. GIAS `phase_code` when it is a real value (not the not-applicable code);
|
||||
2. statutory-age inference emits the matching GIAS code
|
||||
(Primary / Secondary / All-through — numeric values confirmed from
|
||||
live data at implementation);
|
||||
3. school-name heuristics (unchanged — they match `school_name`, which is
|
||||
not one of the six fields) emit the same codes;
|
||||
4. else null.
|
||||
- dbt schema tests: `accepted_values` (severity **warn**) on every code
|
||||
column, values taken from the dictionary; `not_null` warn on `phase_code`
|
||||
(mirrors today's phase test); `has_sixth_form` tests unchanged.
|
||||
|
||||
**`dim_location`**: only the status filter changes (must stay byte-identical
|
||||
to `dim_school`'s — the API inner-joins the two).
|
||||
|
||||
### 4. Dictionaries
|
||||
|
||||
**Canonical module: `backend/gias_codes.py`**
|
||||
|
||||
```python
|
||||
ESTABLISHMENT_STATUS: dict[int, str]
|
||||
SCHOOL_TYPE: dict[int, str]
|
||||
PHASE_OF_EDUCATION: dict[int, str]
|
||||
OFFICIAL_SIXTH_FORM: dict[int, str]
|
||||
RELIGIOUS_CHARACTER: dict[int, str]
|
||||
ADMISSIONS_POLICY: dict[int, str]
|
||||
|
||||
def translate(code: int | None, mapping: dict[int, str]) -> str | None:
|
||||
"""None -> None; unknown code -> 'Unknown (<code>)' + warning log."""
|
||||
```
|
||||
|
||||
- Contents are generated from live raw data
|
||||
(`SELECT DISTINCT code, name FROM raw.gias_establishments ...` per field)
|
||||
and sanity-checked against the DfE GIAS registers. Names must be
|
||||
byte-identical to what the API serves today.
|
||||
- Unknown codes never blank the UI: `translate` returns `"Unknown (<code>)"`
|
||||
and logs, so a new DfE value degrades gracefully.
|
||||
|
||||
**Pipeline copy: `pipeline/scripts/gias_codes.py`**
|
||||
|
||||
The app and pipeline Docker images have disjoint build contexts
|
||||
(`Dockerfile` copies `backend/`; `pipeline/Dockerfile` copies `pipeline/`),
|
||||
so the Typesense sync cannot import the backend module. It gets a
|
||||
byte-identical copy, and a backend unit test asserts
|
||||
`backend/gias_codes.py` and `pipeline/scripts/gias_codes.py` have identical
|
||||
content — drift fails CI. (Deliberately chosen over codegen: six dicts do
|
||||
not justify build machinery.)
|
||||
|
||||
**Seed for drift detection: `pipeline/transform/seeds/gias_code_names.csv`**
|
||||
|
||||
Columns `field,code,name` mirroring the dictionary. A dbt test (severity
|
||||
warn) compares live raw `(code, name)` pairs against the seed; when DfE adds
|
||||
or renames a value the nightly run warns, prompting a dictionary + seed
|
||||
update in one PR.
|
||||
|
||||
### 5. Backend translation (API contract unchanged)
|
||||
|
||||
- `_MAIN_QUERY` selects the code columns instead of the name columns.
|
||||
- `load_school_data_as_dataframe()` translates immediately after
|
||||
`pd.read_sql`, writing today's column names:
|
||||
|
||||
```python
|
||||
df["phase"] = df["phase_code"].map(...)
|
||||
df["school_type"] = df["school_type_code"].map(...) # then normalize_school_type as today
|
||||
df["status"] = df["status_code"].map(...)
|
||||
df["religious_denomination"] = df["religious_character_code"].map(...)
|
||||
df["admissions_policy"] = df["admissions_policy_code"].map(...)
|
||||
```
|
||||
|
||||
Everything downstream — `PHASE_GROUPS`, filters, payload builders,
|
||||
`/api/filters`, frontend, e2e — sees exactly today's strings. No frontend
|
||||
changes.
|
||||
|
||||
- `backend/models.py` `DimSchool`: string columns replaced by
|
||||
`*_code = Column(Integer)`.
|
||||
|
||||
### 6. Typesense sync
|
||||
|
||||
`pipeline/scripts/sync_typesense.py` selects `phase`, `school_type`,
|
||||
`religious_character` today. It switches to the code columns and translates
|
||||
via `pipeline/scripts/gias_codes.py` before indexing, so facet values in
|
||||
search are unchanged.
|
||||
|
||||
### 7. Rollout
|
||||
|
||||
- No DB migration: marts are full-rebuild tables.
|
||||
- Deploy window: until the first post-merge pipeline run, the old marts
|
||||
still carry string columns while the new backend queries code columns, so
|
||||
the backend's query fails and it serves empty data (the one-column retry
|
||||
built for `has_sixth_form` doesn't generalise to six columns, and a full
|
||||
old-schema fallback query isn't worth it). **Decision: accept the window
|
||||
and close it operationally — the runbook is merge → deploy → trigger
|
||||
`school_data_daily` immediately.** The DAG's final step already calls
|
||||
`/api/admin/reload`, so the backend recovers without a restart.
|
||||
- Tests: backend unit tests for `translate()` (known / unknown / None),
|
||||
payload tests asserting names still served, the file-parity test, dbt
|
||||
schema/seed tests. Frontend: no changes; existing Jest suite is the
|
||||
regression net.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Recoding other string columns (`gender`, `urban_rural`,
|
||||
`nursery_provision`, `local_authority_name` …) — same pattern can follow
|
||||
later if this proves out.
|
||||
- Collapsing academy subtypes (today's `normalize_school_type`) — kept
|
||||
as-is, applied after translation.
|
||||
- Serving codes through the API — the contract deliberately keeps names.
|
||||
@@ -25,16 +25,6 @@ test('home page loads with hero search', async ({ page }) => {
|
||||
await expect(page.getByPlaceholder('School name or postcode').first()).toBeVisible();
|
||||
});
|
||||
|
||||
test('home hero offers a "use my location" shortcut beside the search box', async ({ page }) => {
|
||||
await page.goto('/');
|
||||
// The geolocation shortcut lives inside the hero search card, right under the
|
||||
// search input — not in a separate strip further down the page.
|
||||
const searchInput = page.getByPlaceholder('School name or postcode').first();
|
||||
await expect(searchInput).toBeVisible();
|
||||
const nearMe = page.getByRole('button', { name: /use my location/i });
|
||||
await expect(nearMe).toBeVisible();
|
||||
});
|
||||
|
||||
test('searching by name returns school results', async ({ page }) => {
|
||||
await searchByName(page, 'primary');
|
||||
await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 });
|
||||
@@ -60,84 +50,6 @@ test('school detail page renders name and performance data', async ({ page }) =>
|
||||
await expect(page.locator('canvas:visible').first()).toBeVisible({ timeout: 15_000 });
|
||||
});
|
||||
|
||||
test('school with no performance data still gets a working detail page', async ({ page }) => {
|
||||
// Schools without KS2/KS4 results (special post-16 institutions, sixth-form
|
||||
// centres, PRUs) used to 500 in the API — NaN GIAS fields broke JSON
|
||||
// serialization — which the frontend rendered as a 404 on every such SEO
|
||||
// landing page. Find one via the search API (year === null marks "no
|
||||
// performance rows") and assert its page renders.
|
||||
const candidates: number[] = [];
|
||||
for (const q of ['post 16', 'specialist college', 'sixth form']) {
|
||||
const resp = await page.request.get(
|
||||
`/api/schools?search=${encodeURIComponent(q)}&per_page=20`
|
||||
);
|
||||
if (!resp.ok()) continue;
|
||||
const body = await resp.json();
|
||||
for (const s of body.schools ?? []) {
|
||||
if (s.year === null && s.urn) candidates.push(s.urn);
|
||||
}
|
||||
if (candidates.length) break;
|
||||
}
|
||||
test.skip(candidates.length === 0, 'no results-less school in this dataset');
|
||||
|
||||
const detail = await page.request.get(`/api/schools/${candidates[0]}`);
|
||||
expect(detail.status(), 'detail API must not 500 for a results-less school').toBe(200);
|
||||
|
||||
await page.goto(`/school/${candidates[0]}`);
|
||||
await page.waitForURL(/\/school\/\d+-/); // redirected to canonical slug
|
||||
await expect(page.locator('h1').first()).toBeVisible();
|
||||
});
|
||||
|
||||
test('school hero map opens fullscreen on mobile without the Fullscreen API', async ({ page }) => {
|
||||
// iOS Safari has no Element.requestFullscreen; the map must fall back to a
|
||||
// CSS overlay. Simulate that by removing the API before any page script runs.
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.addInitScript(() => {
|
||||
// @ts-expect-error deliberate API removal
|
||||
delete Element.prototype.requestFullscreen;
|
||||
});
|
||||
|
||||
await searchByName(page, 'primary');
|
||||
const firstSchool = schoolLinks(page).first();
|
||||
await expect(firstSchool).toBeVisible({ timeout: 15_000 });
|
||||
await firstSchool.click();
|
||||
await page.waitForURL(/\/school\//);
|
||||
|
||||
const openMap = page.getByRole('button', { name: 'Open full map' });
|
||||
await expect(openMap).toBeVisible({ timeout: 15_000 });
|
||||
await openMap.click();
|
||||
|
||||
const closeMap = page.getByRole('button', { name: 'Close map' });
|
||||
await expect(closeMap).toBeVisible();
|
||||
await closeMap.click();
|
||||
await expect(openMap).toBeVisible();
|
||||
});
|
||||
|
||||
test('results map fullscreen falls back to an overlay on iOS', async ({ page }) => {
|
||||
// Same iOS gap as the hero map: no Element.requestFullscreen, so the results
|
||||
// map's fullscreen button must fall back to a CSS overlay.
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.addInitScript(() => {
|
||||
// @ts-expect-error deliberate API removal
|
||||
delete Element.prototype.requestFullscreen;
|
||||
});
|
||||
|
||||
await searchByName(page, 'B1 1BB');
|
||||
await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// Switch to the map view, then open the map fullscreen.
|
||||
await page.getByRole('button', { name: 'Map', exact: true }).click();
|
||||
const openFs = page.getByRole('button', { name: 'View map fullscreen' });
|
||||
await expect(openFs).toBeVisible({ timeout: 15_000 });
|
||||
await openFs.click();
|
||||
|
||||
// The button flips to its exit state once the overlay is up.
|
||||
const exitFs = page.getByRole('button', { name: 'Exit fullscreen' });
|
||||
await expect(exitFs).toBeVisible();
|
||||
await exitFs.click();
|
||||
await expect(openFs).toBeVisible();
|
||||
});
|
||||
|
||||
test('comparing two schools shows both side by side', async ({ page }) => {
|
||||
// Collect two school URNs from search results, then load the share URL
|
||||
await searchByName(page, 'primary');
|
||||
@@ -154,37 +66,6 @@ test('comparing two schools shows both side by side', async ({ page }) => {
|
||||
await expect(page.locator(`a[href*="${urns[1]}"]`).first()).toBeVisible();
|
||||
});
|
||||
|
||||
test('compare chart on mobile shows school chips with tap-to-focus', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
|
||||
await searchByName(page, 'primary');
|
||||
await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 });
|
||||
const hrefs = await schoolLinks(page).evaluateAll((links) =>
|
||||
links.map((l) => (l as HTMLAnchorElement).getAttribute('href') || '')
|
||||
);
|
||||
const urns = [...new Set(hrefs.map((h) => h.match(/\/school\/(\d+)/)?.[1]).filter(Boolean))];
|
||||
// Compare three schools, not two: a "primary" search can return all-through
|
||||
// schools that classify as secondary, and the chips only appear for the
|
||||
// active phase. With three schools across two phases, the auto-selected
|
||||
// majority phase always holds ≥2, so the chip legend is guaranteed to render.
|
||||
expect(urns.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
await page.goto(`/compare?urns=${urns[0]},${urns[1]},${urns[2]}`);
|
||||
await expect(page.locator('canvas:visible').first()).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// The mobile chart legend renders one chip per school in the active phase.
|
||||
const chipGroup = page.getByRole('group', { name: /highlight a school/i });
|
||||
const chips = chipGroup.getByRole('button');
|
||||
await expect(chips.first()).toBeVisible({ timeout: 15_000 });
|
||||
expect(await chips.count()).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Tapping a chip focuses that school's line; tapping again releases it.
|
||||
await chips.first().click();
|
||||
await expect(chips.first()).toHaveAttribute('aria-pressed', 'true');
|
||||
await chips.first().click();
|
||||
await expect(chips.first()).toHaveAttribute('aria-pressed', 'false');
|
||||
});
|
||||
|
||||
test('rankings page loads a populated table', async ({ page }) => {
|
||||
await page.goto('/rankings');
|
||||
await expect(page.getByRole('heading', { name: /rankings/i }).first()).toBeVisible();
|
||||
@@ -192,24 +73,3 @@ test('rankings page loads a populated table', async ({ page }) => {
|
||||
await expect(rows.first()).toBeVisible({ timeout: 15_000 });
|
||||
expect(await rows.count()).toBeGreaterThan(5);
|
||||
});
|
||||
|
||||
test('rankings stay populated after picking a specific year', async ({ page }) => {
|
||||
// Years are academic-year codes (e.g. 201819); the API must accept them
|
||||
// as the `year` query param rather than rejecting with a 422.
|
||||
await page.goto('/rankings');
|
||||
const yearSelect = page.locator('#year-select');
|
||||
await expect(yearSelect).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// Pick the last option — the most recent explicit year. The default view
|
||||
// already proved this year has rows, so an empty table after selecting it
|
||||
// can only mean the year param was rejected. (The oldest year is no good
|
||||
// here: staging doesn't always carry the full data history.)
|
||||
const yearValue = await yearSelect.locator('option').last().getAttribute('value');
|
||||
expect(yearValue).toBeTruthy();
|
||||
await yearSelect.selectOption(yearValue!);
|
||||
await page.waitForURL(/year=/);
|
||||
|
||||
const rows = page.locator('table tbody tr');
|
||||
await expect(rows.first()).toBeVisible({ timeout: 15_000 });
|
||||
expect(await rows.count()).toBeGreaterThan(5);
|
||||
});
|
||||
|
||||
@@ -22,9 +22,7 @@ COPY . .
|
||||
ENV NEXT_TELEMETRY_DISABLED=1
|
||||
ENV NODE_ENV=production
|
||||
|
||||
# Default backend URL for any server-side fetch during `next build`. The
|
||||
# runtime /api proxy reads FASTAPI_URL per request (see app/api/[...path]),
|
||||
# so the deployed container's env is what actually routes traffic.
|
||||
# Build argument for FastAPI URL (used by Next.js rewrites at build time)
|
||||
ARG FASTAPI_URL=http://backend:80/api
|
||||
ENV FASTAPI_URL=${FASTAPI_URL}
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
/**
|
||||
* SecondarySchoolRow — sixth-form tag must come from the GIAS
|
||||
* has_sixth_form flag, not the age_range-contains-"18" heuristic.
|
||||
*/
|
||||
|
||||
import '@testing-library/jest-dom';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { SecondarySchoolRow } from '@/components/SecondarySchoolRow';
|
||||
import type { School } from '@/lib/types';
|
||||
|
||||
const base = {
|
||||
urn: 100002,
|
||||
school_name: 'Beta Sixth Form College',
|
||||
local_authority: 'Testshire',
|
||||
school_type: 'Academy',
|
||||
phase: 'Secondary',
|
||||
gender: 'Mixed',
|
||||
attainment_8_score: 50.0,
|
||||
} as unknown as School;
|
||||
|
||||
describe('SecondarySchoolRow sixth-form tag', () => {
|
||||
it('shows the tag for a 16-19 college with the GIAS flag set', () => {
|
||||
render(
|
||||
<SecondarySchoolRow
|
||||
school={{ ...base, age_range: '16-19', has_sixth_form: true }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText('Sixth form')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the tag for an 11-18 school without a registered sixth form', () => {
|
||||
render(
|
||||
<SecondarySchoolRow
|
||||
school={{ ...base, age_range: '11-18', has_sixth_form: false }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.queryByText('Sixth form')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the tag when the flag is missing (pipeline not yet re-run)', () => {
|
||||
render(
|
||||
<SecondarySchoolRow school={{ ...base, age_range: '11-18' }} />,
|
||||
);
|
||||
expect(screen.queryByText('Sixth form')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('SecondarySchoolRow proposed-to-close tag', () => {
|
||||
it('shows the tag when GIAS status is "Open, but proposed to close"', () => {
|
||||
render(
|
||||
<SecondarySchoolRow
|
||||
school={{ ...base, status: 'Open, but proposed to close' }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText(/Proposed to close/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the tag for a plain open school', () => {
|
||||
render(<SecondarySchoolRow school={{ ...base, status: 'Open' }} />);
|
||||
expect(screen.queryByText(/Proposed to close/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the tag when status is missing', () => {
|
||||
render(<SecondarySchoolRow school={base} />);
|
||||
expect(screen.queryByText(/Proposed to close/)).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -9,8 +9,6 @@ import {
|
||||
isValidPostcode,
|
||||
debounce,
|
||||
buildOfstedListBadge,
|
||||
metricKind,
|
||||
computeYBounds,
|
||||
} from '@/lib/utils';
|
||||
|
||||
describe('formatPercentage', () => {
|
||||
@@ -161,65 +159,3 @@ describe('buildOfstedListBadge', () => {
|
||||
expect(badge.cssClass).toBe('ofstedPending');
|
||||
});
|
||||
});
|
||||
|
||||
describe('metricKind', () => {
|
||||
it('classifies metrics by key', () => {
|
||||
expect(metricKind('rwm_expected_pct')).toBe('percentage');
|
||||
expect(metricKind('absence_rate')).toBe('percentage');
|
||||
expect(metricKind('reading_progress')).toBe('progress');
|
||||
expect(metricKind('progress_8_score')).toBe('progress');
|
||||
expect(metricKind('attainment_8_score')).toBe('score');
|
||||
expect(metricKind('reading_avg_score')).toBe('score');
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeYBounds', () => {
|
||||
it('tightens clustered percentages instead of framing 0-100', () => {
|
||||
const b = computeYBounds([86, 86, 86, 80, 96], 'percentage');
|
||||
expect(b.min).toBeGreaterThanOrEqual(0);
|
||||
expect(b.max).toBeLessThanOrEqual(100);
|
||||
expect(b.min).toBeGreaterThan(50);
|
||||
expect(b.max! - b.min!).toBeGreaterThanOrEqual(10);
|
||||
});
|
||||
|
||||
it('never widens percentages beyond 0-100 for non-negative data', () => {
|
||||
const b = computeYBounds([2, 5, 98], 'percentage');
|
||||
expect(b.min).toBe(0);
|
||||
expect(b.max).toBe(100);
|
||||
});
|
||||
|
||||
it('does not clamp to zero when pct-named trend data is negative', () => {
|
||||
const b = computeYBounds([-12, -3, 4], 'percentage');
|
||||
expect(b.min).toBeLessThan(-12);
|
||||
});
|
||||
|
||||
it('keeps progress bounds symmetric around zero', () => {
|
||||
const b = computeYBounds([-1.2, 0.4, 2.1], 'progress');
|
||||
expect(b.min).toBe(-b.max!);
|
||||
expect(b.min).toBeLessThanOrEqual(-1.2);
|
||||
expect(b.max).toBeGreaterThanOrEqual(2.1);
|
||||
});
|
||||
|
||||
it('fits score metrics without a fixed frame', () => {
|
||||
const b = computeYBounds([42.3, 48.9, 51.2], 'score');
|
||||
expect(b.min).toBeGreaterThanOrEqual(0);
|
||||
expect(b.min).toBeLessThanOrEqual(42.3);
|
||||
expect(b.max).toBeGreaterThanOrEqual(51.2);
|
||||
});
|
||||
|
||||
it('returns empty bounds when there is no numeric data', () => {
|
||||
expect(computeYBounds([null, undefined, NaN], 'percentage')).toEqual({});
|
||||
expect(computeYBounds([], 'progress')).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('isProposedToClose', () => {
|
||||
const { isProposedToClose } = require('@/lib/utils');
|
||||
|
||||
it('is true only for the exact GIAS proposed-to-close status', () => {
|
||||
expect(isProposedToClose({ status: 'Open, but proposed to close' })).toBe(true);
|
||||
expect(isProposedToClose({ status: 'Open' })).toBe(false);
|
||||
expect(isProposedToClose({ status: null })).toBe(false);
|
||||
expect(isProposedToClose({})).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
/**
|
||||
* Runtime proxy for /api/* → the FastAPI backend.
|
||||
*
|
||||
* This replaces the old next.config.js `rewrites()` proxy, whose destination
|
||||
* was baked into the build (routes-manifest.json) from FASTAPI_URL at build
|
||||
* time. Because one frontend image is promoted staging→prod, a baked hostname
|
||||
* forced every environment to name the backend identically; a mismatch (e.g.
|
||||
* a `backend_stg` service) produced `getaddrinfo ENOTFOUND backend`.
|
||||
*
|
||||
* A route handler reads process.env.FASTAPI_URL on each request, so the same
|
||||
* image adapts to whatever the backend is called in each environment.
|
||||
*/
|
||||
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
// FASTAPI_URL already includes the `/api` suffix (e.g. http://backend:80/api).
|
||||
function backendBase(): string {
|
||||
return process.env.FASTAPI_URL || process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000/api';
|
||||
}
|
||||
|
||||
// Hop-by-hop / length headers must not be copied across a proxy — undici has
|
||||
// already decoded the body, so a stale content-encoding/length corrupts it.
|
||||
const STRIPPED_RESPONSE_HEADERS = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'];
|
||||
const METHODS_WITH_BODY = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
||||
|
||||
async function handler(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
|
||||
const { path } = await ctx.params;
|
||||
const target = `${backendBase()}/${path.join('/')}${req.nextUrl.search}`;
|
||||
|
||||
const headers = new Headers(req.headers);
|
||||
headers.delete('host');
|
||||
headers.delete('connection');
|
||||
|
||||
const init: RequestInit & { duplex?: 'half' } = {
|
||||
method: req.method,
|
||||
headers,
|
||||
redirect: 'manual',
|
||||
cache: 'no-store',
|
||||
};
|
||||
if (METHODS_WITH_BODY.has(req.method)) {
|
||||
init.body = req.body;
|
||||
init.duplex = 'half';
|
||||
}
|
||||
|
||||
let upstream: Response;
|
||||
try {
|
||||
upstream = await fetch(target, init);
|
||||
} catch (err) {
|
||||
// e.g. DNS failure or connection refused — surface a clean 502 instead of
|
||||
// an opaque proxy crash so callers can degrade gracefully.
|
||||
return NextResponse.json({ detail: 'Upstream request failed' }, { status: 502 });
|
||||
}
|
||||
|
||||
const responseHeaders = new Headers(upstream.headers);
|
||||
for (const h of STRIPPED_RESPONSE_HEADERS) responseHeaders.delete(h);
|
||||
|
||||
return new NextResponse(upstream.body, {
|
||||
status: upstream.status,
|
||||
statusText: upstream.statusText,
|
||||
headers: responseHeaders,
|
||||
});
|
||||
}
|
||||
|
||||
export {
|
||||
handler as GET,
|
||||
handler as HEAD,
|
||||
handler as POST,
|
||||
handler as PUT,
|
||||
handler as PATCH,
|
||||
handler as DELETE,
|
||||
handler as OPTIONS,
|
||||
};
|
||||
@@ -133,7 +133,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
const { school_info, yearly_data, absence_data, ofsted, census, admissions, admissions_history, sen_detail, phonics, deprivation, finance } = data;
|
||||
const { school_info, yearly_data, absence_data, ofsted, parent_view, census, admissions, admissions_history, sen_detail, phonics, deprivation, finance } = data;
|
||||
|
||||
// Redirect bare URN to canonical slug URL
|
||||
const canonicalSlug = schoolUrl(urn, school_info.school_name).replace('/school/', '');
|
||||
@@ -189,6 +189,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
yearlyData={yearly_data}
|
||||
absenceData={absence_data}
|
||||
ofsted={ofsted ?? null}
|
||||
parentView={parent_view ?? null}
|
||||
census={census ?? null}
|
||||
admissions={admissions ?? null}
|
||||
senDetail={sen_detail ?? null}
|
||||
@@ -202,6 +203,7 @@ export default async function SchoolPage({ params }: SchoolPageProps) {
|
||||
yearlyData={yearly_data}
|
||||
absenceData={absence_data}
|
||||
ofsted={ofsted ?? null}
|
||||
parentView={parent_view ?? null}
|
||||
census={census ?? null}
|
||||
admissions={admissions ?? null}
|
||||
admissionsHistory={admissions_history ?? []}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
/**
|
||||
* Runtime proxy for /sitemap.xml → the FastAPI backend's generated sitemap.
|
||||
*
|
||||
* Like the /api/* proxy, this reads FASTAPI_URL at request time rather than
|
||||
* baking the backend host into the build, so one image works in every
|
||||
* environment. robots.ts points crawlers here.
|
||||
*/
|
||||
|
||||
import { NextResponse } from 'next/server';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
function backendOrigin(): string {
|
||||
const base = process.env.FASTAPI_URL || process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000/api';
|
||||
return base.replace(/\/api$/, '');
|
||||
}
|
||||
|
||||
export async function GET() {
|
||||
let upstream: Response;
|
||||
try {
|
||||
upstream = await fetch(`${backendOrigin()}/sitemap.xml`, { cache: 'no-store' });
|
||||
} catch {
|
||||
return new NextResponse('Sitemap temporarily unavailable', { status: 502 });
|
||||
}
|
||||
|
||||
const body = await upstream.text();
|
||||
return new NextResponse(body, {
|
||||
status: upstream.status,
|
||||
headers: { 'content-type': upstream.headers.get('content-type') || 'application/xml' },
|
||||
});
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
/* Chart wrapper: chips (mobile) above, canvas filling the rest of the
|
||||
parent .chartContainer, whose fixed height drives Chart.js sizing via
|
||||
maintainAspectRatio: false. */
|
||||
.wrapper {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.canvasBox {
|
||||
position: relative;
|
||||
flex: 1 1 auto;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
/* School chips: mobile-only legend + tap-to-focus control. Desktop keeps
|
||||
Chart.js's built-in legend (with per-school point shapes). */
|
||||
.chips {
|
||||
display: none;
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.chips {
|
||||
/* Two chips per row so long school names don't crowd into a single
|
||||
line; each chip fills its column and truncates with an ellipsis. */
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 6px;
|
||||
padding-bottom: 8px;
|
||||
}
|
||||
|
||||
.chip {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-height: 40px;
|
||||
min-width: 0;
|
||||
padding: 4px 10px;
|
||||
border: 1px solid rgba(0, 0, 0, .12);
|
||||
border-radius: 999px;
|
||||
background: transparent;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.chip[aria-pressed="true"] {
|
||||
background: rgba(0, 0, 0, .06);
|
||||
border-color: rgba(0, 0, 0, .35);
|
||||
}
|
||||
|
||||
.chipDot {
|
||||
flex: 0 0 auto;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
}
|
||||
|
||||
.chipName {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
/* min-width:0 lets the name shrink inside the grid cell so the
|
||||
ellipsis kicks in instead of overflowing. */
|
||||
min-width: 0;
|
||||
}
|
||||
}
|
||||
@@ -1,82 +1,47 @@
|
||||
/**
|
||||
* ComparisonChart Component
|
||||
* Multi-school comparison chart using Chart.js.
|
||||
*
|
||||
* Desktop: built-in legend (point-style markers double as per-school shapes).
|
||||
* Mobile (≤640px): the in-chart legend and axis titles are dropped in favour
|
||||
* of a chip row above the canvas; tapping a chip highlights that school's
|
||||
* line and dims the rest. The y-axis auto-fits the data on all viewports so
|
||||
* clustered schools stay distinguishable.
|
||||
* Multi-school comparison chart using Chart.js
|
||||
*/
|
||||
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { Line } from 'react-chartjs-2';
|
||||
import { ChartOptions, ChartDataset, PointStyle } from 'chart.js';
|
||||
import { ChartOptions } from 'chart.js';
|
||||
import '@/lib/chartSetup';
|
||||
import type { ComparisonData } from '@/lib/types';
|
||||
import {
|
||||
CHART_COLORS,
|
||||
CHART_TEXT_COLORS,
|
||||
computeYBounds,
|
||||
formatAcademicYear,
|
||||
metricKind,
|
||||
rgbToRgba,
|
||||
} from '@/lib/utils';
|
||||
import { useIsMobile } from '@/hooks/useIsMobile';
|
||||
import { track } from '@/lib/analytics';
|
||||
import styles from './ComparisonChart.module.css';
|
||||
import { CHART_COLORS, formatAcademicYear } from '@/lib/utils';
|
||||
|
||||
interface ComparisonChartProps {
|
||||
comparisonData: Record<string, ComparisonData>;
|
||||
/** Ordered as displayed in the school cards, so colours match by index. */
|
||||
schools: Array<{ urn: number; school_name: string }>;
|
||||
metric: string;
|
||||
metricLabel: string;
|
||||
}
|
||||
|
||||
// One shape per basket slot (MAX_SCHOOLS = 5) — secondary encoding so
|
||||
// converging lines stay tellable apart without relying on hue alone.
|
||||
const POINT_STYLES: PointStyle[] = ['circle', 'triangle', 'rect', 'rectRot', 'star'];
|
||||
|
||||
export function ComparisonChart({ comparisonData, schools, metric, metricLabel }: ComparisonChartProps) {
|
||||
const isMobile = useIsMobile();
|
||||
const [focusedUrn, setFocusedUrn] = useState<number | null>(null);
|
||||
|
||||
// A focused school that leaves the basket must not linger.
|
||||
const urnKey = schools.map((s) => s.urn).join(',');
|
||||
useEffect(() => {
|
||||
setFocusedUrn(null);
|
||||
}, [urnKey]);
|
||||
export function ComparisonChart({ comparisonData, metric, metricLabel }: ComparisonChartProps) {
|
||||
// Get all schools and their data
|
||||
const schools = Object.entries(comparisonData);
|
||||
|
||||
if (schools.length === 0) {
|
||||
return <div>No data available</div>;
|
||||
}
|
||||
|
||||
// Union of years across all schools — coverage differs between them.
|
||||
const years = [
|
||||
...new Set(schools.flatMap((s) => comparisonData[String(s.urn)]?.yearly_data.map((d) => d.year) ?? [])),
|
||||
].sort((a, b) => a - b);
|
||||
// Get years from first school (assuming all schools have same years)
|
||||
const years = schools[0][1].yearly_data.map((d) => d.year).sort((a, b) => a - b);
|
||||
|
||||
const datasets: ChartDataset<'line'>[] = schools.map((school, index) => {
|
||||
const data = comparisonData[String(school.urn)];
|
||||
// Create datasets for each school
|
||||
const datasets = schools.map(([urn, data], index) => {
|
||||
const schoolInfo = data.school_info;
|
||||
const color = CHART_COLORS[index % CHART_COLORS.length];
|
||||
const dimmed = focusedUrn !== null && focusedUrn !== school.urn;
|
||||
|
||||
return {
|
||||
label: school.school_name,
|
||||
label: schoolInfo.school_name,
|
||||
data: years.map((year) => {
|
||||
const yearData = data?.yearly_data.find((d) => d.year === year);
|
||||
const yearData = data.yearly_data.find((d) => d.year === year);
|
||||
if (!yearData) return null;
|
||||
return yearData[metric as keyof typeof yearData] as number | null;
|
||||
}),
|
||||
borderColor: dimmed ? rgbToRgba(color, 0.2) : color,
|
||||
backgroundColor: dimmed ? 'transparent' : rgbToRgba(color, 0.1),
|
||||
borderWidth: focusedUrn === school.urn ? 3 : dimmed ? 1.5 : 2,
|
||||
pointStyle: POINT_STYLES[index % POINT_STYLES.length],
|
||||
pointRadius: dimmed ? 2 : isMobile ? 3 : 4,
|
||||
pointHoverRadius: isMobile ? 5 : 6,
|
||||
borderColor: color,
|
||||
backgroundColor: color.replace('rgb', 'rgba').replace(')', ', 0.1)'),
|
||||
tension: 0.3,
|
||||
spanGaps: true,
|
||||
};
|
||||
@@ -87,11 +52,9 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
|
||||
datasets,
|
||||
};
|
||||
|
||||
const kind = metricKind(metric);
|
||||
const yBounds = computeYBounds(
|
||||
datasets.flatMap((ds) => ds.data as Array<number | null>),
|
||||
kind,
|
||||
);
|
||||
// Determine if metric is a progress score or percentage
|
||||
const isProgressScore = metric.includes('progress');
|
||||
const isPercentage = metric.includes('pct') || metric.includes('rate');
|
||||
|
||||
const options: ChartOptions<'line'> = {
|
||||
responsive: true,
|
||||
@@ -102,7 +65,6 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
|
||||
},
|
||||
plugins: {
|
||||
legend: {
|
||||
display: !isMobile,
|
||||
position: 'top' as const,
|
||||
labels: {
|
||||
usePointStyle: true,
|
||||
@@ -112,22 +74,26 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
|
||||
},
|
||||
},
|
||||
},
|
||||
// No in-chart title: the section heading and metric selector above the
|
||||
// chart already state the metric.
|
||||
title: {
|
||||
display: false,
|
||||
display: true,
|
||||
text: `${metricLabel} - Comparison`,
|
||||
font: {
|
||||
size: 16,
|
||||
weight: 'bold',
|
||||
},
|
||||
padding: {
|
||||
bottom: 20,
|
||||
},
|
||||
},
|
||||
tooltip: {
|
||||
backgroundColor: 'rgba(0, 0, 0, 0.8)',
|
||||
padding: isMobile ? 10 : 12,
|
||||
padding: 12,
|
||||
titleFont: {
|
||||
size: isMobile ? 12 : 14,
|
||||
size: 14,
|
||||
},
|
||||
bodyFont: {
|
||||
size: isMobile ? 11 : 13,
|
||||
size: 13,
|
||||
},
|
||||
usePointStyle: true,
|
||||
itemSort: (a, b) => (b.parsed.y ?? -Infinity) - (a.parsed.y ?? -Infinity),
|
||||
callbacks: {
|
||||
label: function (context) {
|
||||
let label = context.dataset.label || '';
|
||||
@@ -135,7 +101,13 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
|
||||
label += ': ';
|
||||
}
|
||||
if (context.parsed.y !== null) {
|
||||
label += context.parsed.y.toFixed(1) + (kind === 'percentage' ? '%' : '');
|
||||
if (isProgressScore) {
|
||||
label += context.parsed.y.toFixed(1);
|
||||
} else if (isPercentage) {
|
||||
label += context.parsed.y.toFixed(1) + '%';
|
||||
} else {
|
||||
label += context.parsed.y.toFixed(1);
|
||||
}
|
||||
} else {
|
||||
label += 'N/A';
|
||||
}
|
||||
@@ -149,18 +121,17 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
|
||||
type: 'linear' as const,
|
||||
display: true,
|
||||
title: {
|
||||
display: !isMobile,
|
||||
text: kind === 'percentage' ? 'Percentage (%)' : kind === 'progress' ? 'Progress Score' : 'Value',
|
||||
display: true,
|
||||
text: isPercentage ? 'Percentage (%)' : isProgressScore ? 'Progress Score' : 'Value',
|
||||
font: {
|
||||
size: 12,
|
||||
weight: 'bold',
|
||||
},
|
||||
},
|
||||
...yBounds,
|
||||
ticks: {
|
||||
font: { size: isMobile ? 10 : 12 },
|
||||
...(isMobile && { maxTicksLimit: 5 }),
|
||||
},
|
||||
...(isPercentage && {
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
grid: {
|
||||
color: 'rgba(0, 0, 0, 0.05)',
|
||||
},
|
||||
@@ -170,58 +141,16 @@ export function ComparisonChart({ comparisonData, schools, metric, metricLabel }
|
||||
display: false,
|
||||
},
|
||||
title: {
|
||||
display: !isMobile,
|
||||
display: true,
|
||||
text: 'Year',
|
||||
font: {
|
||||
size: 12,
|
||||
weight: 'bold',
|
||||
},
|
||||
},
|
||||
ticks: {
|
||||
font: { size: isMobile ? 10 : 12 },
|
||||
...(isMobile && { maxRotation: 0, autoSkip: true, maxTicksLimit: 4 }),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const toggleFocus = (urn: number) => {
|
||||
const next = focusedUrn === urn ? null : urn;
|
||||
setFocusedUrn(next);
|
||||
if (next !== null) track('compare_focus_school', { urn: next });
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={styles.wrapper}>
|
||||
{/* Mobile legend + focus control; a single series needs no legend. */}
|
||||
{schools.length > 1 && (
|
||||
<div className={styles.chips} role="group" aria-label="Highlight a school on the chart">
|
||||
{schools.map((school, index) => (
|
||||
<button
|
||||
key={school.urn}
|
||||
type="button"
|
||||
className={styles.chip}
|
||||
aria-pressed={focusedUrn === school.urn}
|
||||
onClick={() => toggleFocus(school.urn)}
|
||||
>
|
||||
<span
|
||||
className={styles.chipDot}
|
||||
style={{ background: CHART_COLORS[index % CHART_COLORS.length] }}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<span
|
||||
className={styles.chipName}
|
||||
style={{ color: CHART_TEXT_COLORS[index % CHART_TEXT_COLORS.length] }}
|
||||
>
|
||||
{school.school_name}
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<div className={styles.canvasBox}>
|
||||
<Line data={chartData} options={options} aria-label={`${metricLabel} comparison chart`} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
return <Line data={chartData} options={options} />;
|
||||
}
|
||||
|
||||
@@ -454,10 +454,7 @@
|
||||
}
|
||||
|
||||
.chartContainer {
|
||||
/* Taller than desktop's proportion would suggest: the chip legend row
|
||||
sits inside, and the in-chart title/legend/axis titles are gone, so
|
||||
nearly all of this is plot area. */
|
||||
height: 340px;
|
||||
height: 300px;
|
||||
}
|
||||
|
||||
.comparisonTable {
|
||||
|
||||
@@ -111,10 +111,8 @@ export function ComparisonView({
|
||||
setComparisonData(data.comparison);
|
||||
})
|
||||
.catch((err) => {
|
||||
// Keep whatever we already have (SSR data or a previous fetch) rather
|
||||
// than blanking the chart — a transient refetch failure shouldn't
|
||||
// destroy a working comparison the user is looking at.
|
||||
console.error('Failed to fetch comparison:', err);
|
||||
setComparisonData(null);
|
||||
});
|
||||
} else {
|
||||
setComparisonData(null);
|
||||
@@ -431,7 +429,6 @@ export function ComparisonView({
|
||||
<div className={styles.chartContainer}>
|
||||
<ComparisonChart
|
||||
comparisonData={activeComparisonData}
|
||||
schools={activeSchools}
|
||||
metric={selectedMetric}
|
||||
metricLabel={metricLabel}
|
||||
/>
|
||||
|
||||
@@ -36,91 +36,6 @@
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.searchHint {
|
||||
margin: 0.875rem 0 0;
|
||||
font-size: 0.95rem;
|
||||
color: var(--text-secondary, #5a554d);
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.searchHint strong {
|
||||
color: var(--text-primary, #1a1612);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.searchHint {
|
||||
font-size: 0.85rem;
|
||||
text-align: left;
|
||||
}
|
||||
}
|
||||
|
||||
.nearMeRow {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.nearMeBtn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.625rem 1.375rem;
|
||||
background: var(--accent-teal, #2d7d7d);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 999px;
|
||||
font-size: 0.9375rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s ease, transform 0.15s ease;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.nearMeBtn:hover:not(:disabled) {
|
||||
background: #235f5f;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.nearMeBtn:disabled {
|
||||
opacity: 0.7;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.nearMeSpinner {
|
||||
display: inline-block;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
border: 2px solid rgba(255, 255, 255, 0.35);
|
||||
border-top-color: #fff;
|
||||
border-radius: 50%;
|
||||
animation: nearMeSpin 0.7s linear infinite;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
@keyframes nearMeSpin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.geoError {
|
||||
font-size: 0.8125rem;
|
||||
color: var(--accent-coral-dark, #b04a2e);
|
||||
margin: 0;
|
||||
max-width: 340px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.nearMeBtn {
|
||||
width: 100%;
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
|
||||
.searchSection {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
@@ -11,21 +11,9 @@ interface FilterBarProps {
|
||||
filters: Filters;
|
||||
isHero?: boolean;
|
||||
resultFilters?: ResultFilters;
|
||||
// Geolocation "use my location" affordance, shown beside the hero search box.
|
||||
// The state and handler live in HomeView (which owns the geolocation flow).
|
||||
onNearMe?: () => void;
|
||||
geoState?: "idle" | "requesting" | "error";
|
||||
geoError?: string | null;
|
||||
}
|
||||
|
||||
export function FilterBar({
|
||||
filters,
|
||||
isHero,
|
||||
resultFilters,
|
||||
onNearMe,
|
||||
geoState = "idle",
|
||||
geoError,
|
||||
}: FilterBarProps) {
|
||||
export function FilterBar({ filters, isHero, resultFilters }: FilterBarProps) {
|
||||
const router = useRouter();
|
||||
const pathname = usePathname();
|
||||
const searchParams = useSearchParams();
|
||||
@@ -194,52 +182,6 @@ export function FilterBar({
|
||||
{isPending ? <div className={styles.spinner}></div> : "Search"}
|
||||
</button>
|
||||
</div>
|
||||
{isHero && (
|
||||
<>
|
||||
<p className={styles.searchHint}>
|
||||
Search by <strong>school name</strong> — or use your{" "}
|
||||
<strong>postcode</strong> for the nearest schools.
|
||||
</p>
|
||||
{onNearMe && (
|
||||
<div className={styles.nearMeRow}>
|
||||
<button
|
||||
type="button"
|
||||
className={styles.nearMeBtn}
|
||||
onClick={onNearMe}
|
||||
disabled={geoState === "requesting"}
|
||||
>
|
||||
{geoState === "requesting" ? (
|
||||
<>
|
||||
<span className={styles.nearMeSpinner} aria-hidden="true" />
|
||||
Locating you…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<svg
|
||||
width="15"
|
||||
height="15"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2.5"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path d="M12 2a7 7 0 0 1 7 7c0 5.25-7 13-7 13S5 14.25 5 9a7 7 0 0 1 7-7z" />
|
||||
<circle cx="12" cy="9" r="2.5" />
|
||||
</svg>
|
||||
Use my location
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
{geoError && (
|
||||
<p className={styles.geoError} role="alert">
|
||||
{geoError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</form>
|
||||
|
||||
{!isHero && (
|
||||
@@ -368,8 +310,8 @@ export function FilterBar({
|
||||
disabled={isPending}
|
||||
>
|
||||
<option value="">With or without sixth form</option>
|
||||
<option value="yes">With sixth form</option>
|
||||
<option value="no">Without sixth form</option>
|
||||
<option value="yes">With sixth form (11-18)</option>
|
||||
<option value="no">Without sixth form (11-16)</option>
|
||||
</select>
|
||||
|
||||
{admissionsPolicyOptions.length > 0 && (
|
||||
|
||||
@@ -369,16 +369,6 @@
|
||||
|
||||
.viewToggle {
|
||||
justify-content: center;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
/* The sort <select> sizes to its widest option ("Highest Reading, Writing
|
||||
& Maths %"), which overflows a phone viewport — beside the view toggle it
|
||||
ran off the right edge. Let it flex into the remaining space and shrink;
|
||||
the selected label truncates instead of pushing past the screen. */
|
||||
.sortSelect {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.mapViewContainer {
|
||||
@@ -506,6 +496,68 @@
|
||||
}
|
||||
}
|
||||
|
||||
.discoverySection {
|
||||
padding: 0.5rem 0 0.5rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.nearMeRow {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
|
||||
.nearMeBtn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.625rem 1.375rem;
|
||||
background: var(--accent-teal, #2d7d7d);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 999px;
|
||||
font-size: 0.9375rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s ease, transform 0.15s ease;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.nearMeBtn:hover:not(:disabled) {
|
||||
background: #235f5f;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.nearMeBtn:disabled {
|
||||
opacity: 0.7;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.nearMeBtnSpinner {
|
||||
display: inline-block;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
border: 2px solid rgba(255, 255, 255, 0.35);
|
||||
border-top-color: #fff;
|
||||
border-radius: 50%;
|
||||
animation: nearMeSpin 0.7s linear infinite;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
@keyframes nearMeSpin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
.geoError {
|
||||
font-size: 0.8125rem;
|
||||
color: var(--accent-coral-dark, #b04a2e);
|
||||
margin: 0;
|
||||
max-width: 340px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.quickSearches {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
@@ -284,11 +284,37 @@ export function HomeView({ initialSchools, filters, totalSchools, howItWorks, ed
|
||||
filters={filters}
|
||||
isHero={!isSearchActive}
|
||||
resultFilters={initialSchools.result_filters}
|
||||
onNearMe={handleNearMe}
|
||||
geoState={geoState}
|
||||
geoError={geoError}
|
||||
/>
|
||||
|
||||
{/* Discovery section shown on landing page before any search */}
|
||||
{!isSearchActive && initialSchools.schools.length === 0 && (
|
||||
<div className={styles.discoverySection}>
|
||||
<div className={styles.nearMeRow}>
|
||||
<button
|
||||
className={styles.nearMeBtn}
|
||||
onClick={handleNearMe}
|
||||
disabled={geoState === 'requesting'}
|
||||
>
|
||||
{geoState === 'requesting' ? (
|
||||
<>
|
||||
<span className={styles.nearMeBtnSpinner} aria-hidden="true" />
|
||||
Locating you…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.5" aria-hidden="true">
|
||||
<path d="M12 2a7 7 0 0 1 7 7c0 5.25-7 13-7 13S5 14.25 5 9a7 7 0 0 1 7-7z"/>
|
||||
<circle cx="12" cy="9" r="2.5"/>
|
||||
</svg>
|
||||
Schools near me
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
{geoError && <p className={styles.geoError} role="alert">{geoError}</p>}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Admissions countdown strip — only on landing page */}
|
||||
{!isSearchActive && (
|
||||
<section className={styles.admissionsStrip}>
|
||||
|
||||
@@ -10,13 +10,12 @@
|
||||
|
||||
'use client';
|
||||
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { Line } from 'react-chartjs-2';
|
||||
import { ChartOptions, ChartDataset } from 'chart.js';
|
||||
import '@/lib/chartSetup';
|
||||
import type { SchoolResult } from '@/lib/types';
|
||||
import { formatAcademicYear } from '@/lib/utils';
|
||||
import { useIsMobile } from '@/hooks/useIsMobile';
|
||||
import { track } from '@/lib/analytics';
|
||||
import styles from './PerformanceChart.module.css';
|
||||
|
||||
@@ -69,7 +68,16 @@ export function PerformanceChart({
|
||||
const sortedData = [...data].sort((a, b) => a.year - b.year);
|
||||
const years = sortedData.map(d => formatAcademicYear(d.year));
|
||||
|
||||
const isMobile = useIsMobile();
|
||||
// ── Mobile detection ─────────────────────────────────────────────────
|
||||
// Hydration-safe: SSR renders desktop; client flips to mobile after mount.
|
||||
const [isMobile, setIsMobile] = useState(false);
|
||||
useEffect(() => {
|
||||
const mq = window.matchMedia('(max-width: 640px)');
|
||||
const update = () => setIsMobile(mq.matches);
|
||||
update();
|
||||
mq.addEventListener('change', update);
|
||||
return () => mq.removeEventListener('change', update);
|
||||
}, []);
|
||||
|
||||
// ── Build per-year national averages ─────────────────────────────────
|
||||
const natRefRwm: (number | null)[] = sortedData.map(d => {
|
||||
|
||||
@@ -713,6 +713,18 @@
|
||||
margin: -0.5rem 0 1rem;
|
||||
}
|
||||
|
||||
/* Response count badge */
|
||||
.responseBadge {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
font-family: var(--font-dm-sans), sans-serif;
|
||||
color: var(--text-muted, #8a847a);
|
||||
background: var(--bg-secondary, #f3ede4);
|
||||
padding: 0.1rem 0.5rem;
|
||||
border-radius: 999px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.subSectionTitle {
|
||||
font-size: 0.875rem;
|
||||
font-weight: 600;
|
||||
@@ -720,6 +732,18 @@
|
||||
margin: 1.25rem 0 0.75rem;
|
||||
}
|
||||
|
||||
/* Parent recommendation line in Ofsted section */
|
||||
.parentRecommendLine {
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-secondary, #5c564d);
|
||||
margin: 0.5rem 0 0;
|
||||
}
|
||||
|
||||
.parentRecommendLine strong {
|
||||
color: var(--accent-teal, #2d7d7d);
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
/* Metrics Grid & Cards */
|
||||
.metricsGrid {
|
||||
display: grid;
|
||||
@@ -1070,6 +1094,49 @@
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* Parent View */
|
||||
.parentViewGrid {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.parentViewRow {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.parentViewLabel {
|
||||
flex: 0 0 18rem;
|
||||
color: var(--text-secondary, #5c564d);
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
|
||||
.parentViewBar {
|
||||
flex: 1;
|
||||
height: 0.5rem;
|
||||
background: var(--bg-secondary, #f3ede4);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.parentViewFill {
|
||||
height: 100%;
|
||||
background: var(--accent-teal, #2d7d7d);
|
||||
border-radius: 4px;
|
||||
transition: width 0.4s ease;
|
||||
}
|
||||
|
||||
.parentViewPct {
|
||||
flex: 0 0 2.75rem;
|
||||
text-align: right;
|
||||
font-size: 0.8125rem;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary, #1a1612);
|
||||
}
|
||||
|
||||
/* Admissions badge — uses unified status colours */
|
||||
.admissionsBadge {
|
||||
display: inline-flex;
|
||||
@@ -1202,6 +1269,25 @@
|
||||
}
|
||||
|
||||
@media (max-width: 480px) {
|
||||
.parentViewRow {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.parentViewLabel {
|
||||
flex: none;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.parentViewBar {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.parentViewPct {
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.card {
|
||||
padding: 1rem;
|
||||
}
|
||||
@@ -1543,18 +1629,3 @@
|
||||
.historyDisclosure[open] > .historyToggle::before {
|
||||
transform: rotate(90deg);
|
||||
}
|
||||
|
||||
/* GIAS "Open, but proposed to close" notice strip */
|
||||
.closingStrip {
|
||||
background: #fdf6e3;
|
||||
border-left: 4px solid #e2c96f;
|
||||
border-radius: 0 6px 6px 0;
|
||||
padding: 0.55rem 0.9rem;
|
||||
margin: 0.5rem 0;
|
||||
font-size: 0.88rem;
|
||||
color: #6e5a00;
|
||||
max-width: 68ch;
|
||||
}
|
||||
.closingStrip strong {
|
||||
color: #8a6200;
|
||||
}
|
||||
|
||||
@@ -13,12 +13,12 @@ import { SchoolHeroMap, type SchoolHeroMapHandle } from './SchoolHeroMap';
|
||||
import { MetricTooltip } from './MetricTooltip';
|
||||
import type {
|
||||
School, SchoolResult, AbsenceData,
|
||||
OfstedInspection, SchoolCensus,
|
||||
OfstedInspection, OfstedParentView, SchoolCensus,
|
||||
SchoolAdmissions, SenDetail, Phonics,
|
||||
SchoolDeprivation, SchoolFinance, NationalAverages,
|
||||
} from '@/lib/types';
|
||||
import {
|
||||
formatPercentage, formatProgress, formatAcademicYear, isProposedToClose,
|
||||
formatPercentage, formatProgress, formatAcademicYear,
|
||||
} from '@/lib/utils';
|
||||
import { DeltaChip } from './DeltaChip';
|
||||
|
||||
@@ -63,6 +63,7 @@ interface SchoolDetailViewProps {
|
||||
yearlyData: SchoolResult[];
|
||||
absenceData: AbsenceData | null;
|
||||
ofsted: OfstedInspection | null;
|
||||
parentView: OfstedParentView | null;
|
||||
census: SchoolCensus | null;
|
||||
admissions: SchoolAdmissions | null;
|
||||
admissionsHistory: SchoolAdmissions[];
|
||||
@@ -74,7 +75,7 @@ interface SchoolDetailViewProps {
|
||||
|
||||
export function SchoolDetailView({
|
||||
schoolInfo, yearlyData, absenceData,
|
||||
ofsted, census, admissions, admissionsHistory, senDetail, phonics, deprivation, finance,
|
||||
ofsted, parentView, census, admissions, admissionsHistory, senDetail, phonics, deprivation, finance,
|
||||
}: SchoolDetailViewProps) {
|
||||
const router = useRouter();
|
||||
const { addSchool, removeSchool, isSelected } = useComparison();
|
||||
@@ -233,6 +234,8 @@ export function SchoolDetailView({
|
||||
if (hasInclusionData) navItems.push({ id: 'inclusion', label: 'Pupils' });
|
||||
if (yearlyData.length > 0) navItems.push({ id: 'history', label: 'History' });
|
||||
if (hasPhonics && isPrimary) navItems.push({ id: 'phonics', label: 'Phonics' });
|
||||
if (parentView && parentView.total_responses != null && parentView.total_responses > 0)
|
||||
navItems.push({ id: 'parents', label: 'Parents' });
|
||||
if (hasSchoolLife) navItems.push({ id: 'school-life', label: 'School Life' });
|
||||
if (hasDeprivation) navItems.push({ id: 'local-area', label: 'Local Area' });
|
||||
if (hasFinance) navItems.push({ id: 'finances', label: 'Finances' });
|
||||
@@ -313,12 +316,6 @@ export function SchoolDetailView({
|
||||
<span className={styles.metaItem}>{schoolInfo.gender}'s school</span>
|
||||
)}
|
||||
</div>
|
||||
{isProposedToClose(schoolInfo) && (
|
||||
<div className={styles.closingStrip} role="note">
|
||||
<strong>⚠ Proposed to close</strong> — this school is proposed for closure,
|
||||
check with the local authority before applying.
|
||||
</div>
|
||||
)}
|
||||
{schoolInfo.address && (
|
||||
<p className={styles.address}>
|
||||
{schoolInfo.address}{schoolInfo.postcode && `, ${schoolInfo.postcode}`}
|
||||
@@ -552,6 +549,11 @@ export function SchoolDetailView({
|
||||
) : null;
|
||||
})}
|
||||
</div>
|
||||
{parentView?.q_recommend_pct != null && parentView.total_responses != null && parentView.total_responses > 0 && (
|
||||
<p className={styles.parentRecommendLine}>
|
||||
<strong>{Math.round(parentView.q_recommend_pct)}%</strong> of parents would recommend this school ({parentView.total_responses.toLocaleString()} responses)
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
/* ── Old OEIF layout ── */
|
||||
@@ -570,6 +572,11 @@ export function SchoolDetailView({
|
||||
<p className={styles.ofstedDisclaimer}>
|
||||
From September 2024, Ofsted no longer makes an overall effectiveness judgement in inspections of state-funded schools.
|
||||
</p>
|
||||
{parentView?.q_recommend_pct != null && parentView.total_responses != null && parentView.total_responses > 0 && (
|
||||
<p className={styles.parentRecommendLine}>
|
||||
<strong>{Math.round(parentView.q_recommend_pct)}%</strong> of parents would recommend this school ({parentView.total_responses.toLocaleString()} responses)
|
||||
</p>
|
||||
)}
|
||||
{oeifAllSameGrade ? (
|
||||
<p className={styles.ofstedAllSame}>
|
||||
Rated <strong>{OFSTED_LABELS[ofsted.overall_effectiveness!]}</strong> across all inspected areas — Quality of Teaching, Behaviour, Pupils' Development and Leadership.
|
||||
@@ -1122,6 +1129,42 @@ export function SchoolDetailView({
|
||||
</section>
|
||||
)}
|
||||
|
||||
{/* What Parents Say */}
|
||||
{parentView && parentView.total_responses != null && parentView.total_responses > 0 && (
|
||||
<section id="parents" className={styles.card}>
|
||||
<h2 className={styles.sectionTitle}>
|
||||
What Parents Say
|
||||
<span className={styles.responseBadge}>
|
||||
{parentView.total_responses.toLocaleString()} responses
|
||||
</span>
|
||||
</h2>
|
||||
<p className={styles.sectionSubtitle}>
|
||||
From the Ofsted Parent View survey — parents share their experience of this school.
|
||||
</p>
|
||||
<div className={styles.parentViewGrid}>
|
||||
{[
|
||||
{ label: 'Would recommend this school', pct: parentView.q_recommend_pct },
|
||||
{ label: 'My child is happy here', pct: parentView.q_happy_pct },
|
||||
{ label: 'My child feels safe here', pct: parentView.q_safe_pct },
|
||||
{ label: 'Teaching is good', pct: parentView.q_teaching_pct },
|
||||
{ label: 'My child makes good progress', pct: parentView.q_progress_pct },
|
||||
{ label: 'School looks after pupils\' wellbeing', pct: parentView.q_wellbeing_pct },
|
||||
{ label: 'Behaviour is well managed', pct: parentView.q_behaviour_pct },
|
||||
{ label: 'School deals well with bullying', pct: parentView.q_bullying_pct },
|
||||
{ label: 'Communicates well with parents', pct: parentView.q_communication_pct },
|
||||
].filter(q => q.pct != null).map(({ label, pct }) => (
|
||||
<div key={label} className={styles.parentViewRow}>
|
||||
<span className={styles.parentViewLabel}>{label}</span>
|
||||
<div className={styles.parentViewBar}>
|
||||
<div className={styles.parentViewFill} style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
<span className={styles.parentViewPct}>{Math.round(pct!)}%</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{/* School Life */}
|
||||
{hasSchoolLife && (
|
||||
<section id="school-life" className={styles.card}>
|
||||
|
||||
@@ -34,15 +34,6 @@
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
/* Fallback fullscreen (iOS Safari — no Element.requestFullscreen): the API
|
||||
can't promote the element, so pin it over the page ourselves. Above the
|
||||
comparison toast (3000) and everything else except modals (9999+). */
|
||||
.wrapper[data-fs-fallback] {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
z-index: 5000;
|
||||
}
|
||||
|
||||
.skeleton {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
|
||||
@@ -29,50 +29,25 @@ interface SchoolHeroMapProps {
|
||||
export const SchoolHeroMap = forwardRef<SchoolHeroMapHandle, SchoolHeroMapProps>(
|
||||
function SchoolHeroMap({ lat, lng }, ref) {
|
||||
const wrapperRef = useRef<HTMLDivElement>(null);
|
||||
const [nativeFullscreen, setNativeFullscreen] = useState(false);
|
||||
// iOS Safari has no Element.requestFullscreen — fall back to a
|
||||
// fixed-position overlay driven by state instead of the Fullscreen API.
|
||||
const [fallbackFullscreen, setFallbackFullscreen] = useState(false);
|
||||
const isFullscreen = nativeFullscreen || fallbackFullscreen;
|
||||
const [isFullscreen, setIsFullscreen] = useState(false);
|
||||
|
||||
const open = useCallback(() => {
|
||||
const el = wrapperRef.current;
|
||||
if (!el) return;
|
||||
if (el.requestFullscreen) {
|
||||
el.requestFullscreen().catch(() => setFallbackFullscreen(true));
|
||||
} else {
|
||||
setFallbackFullscreen(true);
|
||||
}
|
||||
wrapperRef.current?.requestFullscreen?.().catch(() => {});
|
||||
}, []);
|
||||
const close = useCallback(() => {
|
||||
if (document.fullscreenElement) document.exitFullscreen().catch(() => {});
|
||||
setFallbackFullscreen(false);
|
||||
}, []);
|
||||
|
||||
useImperativeHandle(ref, () => ({ open }), [open]);
|
||||
|
||||
useEffect(() => {
|
||||
const onChange = () => setNativeFullscreen(!!document.fullscreenElement);
|
||||
const onChange = () => setIsFullscreen(!!document.fullscreenElement);
|
||||
document.addEventListener('fullscreenchange', onChange);
|
||||
return () => document.removeEventListener('fullscreenchange', onChange);
|
||||
}, []);
|
||||
|
||||
// The fallback overlay sits on top of the page rather than replacing it,
|
||||
// so lock body scroll while it is up.
|
||||
useEffect(() => {
|
||||
if (!fallbackFullscreen) return;
|
||||
const prev = document.body.style.overflow;
|
||||
document.body.style.overflow = 'hidden';
|
||||
return () => { document.body.style.overflow = prev; };
|
||||
}, [fallbackFullscreen]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={wrapperRef}
|
||||
className={styles.wrapper}
|
||||
data-fullscreen={isFullscreen || undefined}
|
||||
data-fs-fallback={fallbackFullscreen || undefined}
|
||||
>
|
||||
<div ref={wrapperRef} className={styles.wrapper} data-fullscreen={isFullscreen || undefined}>
|
||||
<LeafletHeroMap lat={lat} lng={lng} interactive={isFullscreen} />
|
||||
|
||||
{isFullscreen ? (
|
||||
|
||||
@@ -10,15 +10,6 @@
|
||||
height: 100dvh;
|
||||
}
|
||||
|
||||
/* Fallback fullscreen (iOS Safari — no Element.requestFullscreen): the API
|
||||
can't promote the element, so pin it over the page ourselves. Above the
|
||||
comparison toast (3000) and the bottom nav; below modals (9999+). */
|
||||
.mapWrapper.fsFallback {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
z-index: 5000;
|
||||
}
|
||||
|
||||
.fullscreenBtn {
|
||||
position: absolute;
|
||||
top: 0.625rem;
|
||||
|
||||
@@ -33,52 +33,22 @@ interface SchoolMapProps {
|
||||
|
||||
export function SchoolMap({ schools, center, zoom = 13, referencePoint, onMarkerClick, nationalAvgRwm, laAverages }: SchoolMapProps) {
|
||||
const wrapperRef = useRef<HTMLDivElement>(null);
|
||||
const [nativeFullscreen, setNativeFullscreen] = useState(false);
|
||||
// iOS Safari has no Element.requestFullscreen — fall back to a fixed-position
|
||||
// overlay driven by state instead of the Fullscreen API.
|
||||
const [fallbackFullscreen, setFallbackFullscreen] = useState(false);
|
||||
const isFullscreen = nativeFullscreen || fallbackFullscreen;
|
||||
const [isFullscreen, setIsFullscreen] = useState(false);
|
||||
|
||||
// Sync state with browser fullscreen events (e.g. Escape key)
|
||||
useEffect(() => {
|
||||
const onFsChange = () => setNativeFullscreen(!!document.fullscreenElement);
|
||||
const onFsChange = () => setIsFullscreen(!!document.fullscreenElement);
|
||||
document.addEventListener('fullscreenchange', onFsChange);
|
||||
return () => document.removeEventListener('fullscreenchange', onFsChange);
|
||||
}, []);
|
||||
|
||||
// Lock body scroll while the fallback overlay is up.
|
||||
useEffect(() => {
|
||||
if (!fallbackFullscreen) return;
|
||||
const prev = document.body.style.overflow;
|
||||
document.body.style.overflow = 'hidden';
|
||||
return () => { document.body.style.overflow = prev; };
|
||||
}, [fallbackFullscreen]);
|
||||
|
||||
// Leaflet re-measures on window resize (trackResize). Native fullscreen fires
|
||||
// one; the CSS fallback overlay changes size without a resize event, so nudge
|
||||
// Leaflet after the layout settles or the map fills only part of the screen.
|
||||
useEffect(() => {
|
||||
const id = requestAnimationFrame(() => window.dispatchEvent(new Event('resize')));
|
||||
return () => cancelAnimationFrame(id);
|
||||
}, [isFullscreen]);
|
||||
|
||||
const toggleFullscreen = useCallback(() => {
|
||||
if (document.fullscreenElement) {
|
||||
document.exitFullscreen().catch(() => {});
|
||||
return;
|
||||
}
|
||||
if (fallbackFullscreen) {
|
||||
setFallbackFullscreen(false);
|
||||
return;
|
||||
}
|
||||
const el = wrapperRef.current;
|
||||
if (!el) return;
|
||||
if (el.requestFullscreen) {
|
||||
el.requestFullscreen().catch(() => setFallbackFullscreen(true));
|
||||
if (!document.fullscreenElement) {
|
||||
wrapperRef.current?.requestFullscreen();
|
||||
} else {
|
||||
setFallbackFullscreen(true);
|
||||
document.exitFullscreen();
|
||||
}
|
||||
}, [fallbackFullscreen]);
|
||||
}, []);
|
||||
|
||||
// Calculate center if not provided
|
||||
const mapCenter: [number, number] = center || (() => {
|
||||
@@ -94,7 +64,7 @@ export function SchoolMap({ schools, center, zoom = 13, referencePoint, onMarker
|
||||
})();
|
||||
|
||||
return (
|
||||
<div ref={wrapperRef} className={`${styles.mapWrapper} ${isFullscreen ? styles.fullscreen : ''} ${fallbackFullscreen ? styles.fsFallback : ''}`}>
|
||||
<div ref={wrapperRef} className={`${styles.mapWrapper} ${isFullscreen ? styles.fullscreen : ''}`}>
|
||||
<button
|
||||
className={styles.fullscreenBtn}
|
||||
onClick={toggleFullscreen}
|
||||
|
||||
@@ -254,10 +254,3 @@
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
|
||||
/* GIAS "Open, but proposed to close" marker */
|
||||
.attrClosing {
|
||||
background: #fdf6e3;
|
||||
color: #8a6200;
|
||||
border: 1px solid #e2c96f;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
*/
|
||||
|
||||
import type { School } from '@/lib/types';
|
||||
import { formatPercentage, calculateTrend, getPhaseStyle, schoolUrl, buildOfstedListBadge, formatAgeRange, isProposedToClose } from '@/lib/utils';
|
||||
import { formatPercentage, calculateTrend, getPhaseStyle, schoolUrl, buildOfstedListBadge, formatAgeRange } from '@/lib/utils';
|
||||
import styles from './SchoolRow.module.css';
|
||||
|
||||
interface SchoolRowProps {
|
||||
@@ -78,9 +78,6 @@ export function SchoolRow({
|
||||
{school.age_range && <span className={styles.attr}>{formatAgeRange(school.age_range)}</span>}
|
||||
{showDenomination && <span className={styles.attr}>{school.religious_denomination}</span>}
|
||||
{showGender && <span className={styles.attr}>{school.gender}</span>}
|
||||
{isProposedToClose(school) && (
|
||||
<span className={`${styles.attr} ${styles.attrClosing}`}>⚠ Proposed to close</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Line 3: Key stats */}
|
||||
|
||||
@@ -383,6 +383,17 @@
|
||||
margin: 1.25rem 0 0.75rem;
|
||||
}
|
||||
|
||||
.responseBadge {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
font-family: var(--font-dm-sans), sans-serif;
|
||||
color: var(--text-muted, #8a847a);
|
||||
background: var(--bg-secondary, #f3ede4);
|
||||
padding: 0.1rem 0.5rem;
|
||||
border-radius: 999px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
/* ── Progress 8 suspension banner ───────────────────── */
|
||||
.p8Banner {
|
||||
background: rgba(180, 120, 0, 0.1);
|
||||
@@ -653,6 +664,60 @@
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* ── Parent View ─────────────────────────────────────── */
|
||||
.parentRecommendLine {
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-secondary, #5c564d);
|
||||
margin: 0.5rem 0 0;
|
||||
}
|
||||
|
||||
.parentRecommendLine strong {
|
||||
color: var(--accent-teal, #2d7d7d);
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.parentViewGrid {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.parentViewRow {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.parentViewLabel {
|
||||
flex: 0 0 18rem;
|
||||
color: var(--text-secondary, #5c564d);
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
|
||||
.parentViewBar {
|
||||
flex: 1;
|
||||
height: 0.5rem;
|
||||
background: var(--bg-secondary, #f3ede4);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.parentViewFill {
|
||||
height: 100%;
|
||||
background: var(--accent-teal, #2d7d7d);
|
||||
border-radius: 4px;
|
||||
transition: width 0.4s ease;
|
||||
}
|
||||
|
||||
.parentViewPct {
|
||||
flex: 0 0 2.75rem;
|
||||
text-align: right;
|
||||
font-size: 0.8125rem;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary, #1a1612);
|
||||
}
|
||||
|
||||
/* ── Admissions ──────────────────────────────────────── */
|
||||
.admissionsTypeBadge {
|
||||
border-radius: 6px;
|
||||
@@ -1070,6 +1135,10 @@
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.parentViewLabel {
|
||||
flex-basis: 10rem;
|
||||
}
|
||||
|
||||
.ofstedReportLink {
|
||||
margin-left: 0;
|
||||
display: block;
|
||||
@@ -1082,6 +1151,25 @@
|
||||
}
|
||||
|
||||
@media (max-width: 480px) {
|
||||
.parentViewRow {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.parentViewLabel {
|
||||
flex: none;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.parentViewBar {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.parentViewPct {
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.metricsGrid {
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 0.5rem;
|
||||
@@ -1099,18 +1187,3 @@
|
||||
padding: 0.75rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* GIAS "Open, but proposed to close" notice strip */
|
||||
.closingStrip {
|
||||
background: #fdf6e3;
|
||||
border-left: 4px solid #e2c96f;
|
||||
border-radius: 0 6px 6px 0;
|
||||
padding: 0.55rem 0.9rem;
|
||||
margin: 0.5rem 0;
|
||||
font-size: 0.88rem;
|
||||
color: #6e5a00;
|
||||
max-width: 68ch;
|
||||
}
|
||||
.closingStrip strong {
|
||||
color: #8a6200;
|
||||
}
|
||||
|
||||
@@ -19,11 +19,11 @@ const PerformanceChart = dynamic(
|
||||
);
|
||||
import type {
|
||||
School, SchoolResult, AbsenceData,
|
||||
OfstedInspection, SchoolCensus,
|
||||
OfstedInspection, OfstedParentView, SchoolCensus,
|
||||
SchoolAdmissions, SenDetail, Phonics,
|
||||
SchoolDeprivation, SchoolFinance, NationalAverages,
|
||||
} from '@/lib/types';
|
||||
import { formatPercentage, formatProgress, formatAcademicYear, formatAgeRange, isProposedToClose } from '@/lib/utils';
|
||||
import { formatPercentage, formatProgress, formatAcademicYear, formatAgeRange } from '@/lib/utils';
|
||||
import { DeltaChip } from './DeltaChip';
|
||||
import { track, getNavigationSource } from '@/lib/analytics';
|
||||
import styles from './SecondarySchoolDetailView.module.css';
|
||||
@@ -65,6 +65,7 @@ interface SecondarySchoolDetailViewProps {
|
||||
yearlyData: SchoolResult[];
|
||||
absenceData: AbsenceData | null;
|
||||
ofsted: OfstedInspection | null;
|
||||
parentView: OfstedParentView | null;
|
||||
census: SchoolCensus | null;
|
||||
admissions: SchoolAdmissions | null;
|
||||
senDetail: SenDetail | null;
|
||||
@@ -75,7 +76,7 @@ interface SecondarySchoolDetailViewProps {
|
||||
|
||||
export function SecondarySchoolDetailView({
|
||||
schoolInfo, yearlyData,
|
||||
ofsted, census, admissions, senDetail, deprivation, finance, absenceData,
|
||||
ofsted, parentView, census, admissions, senDetail, deprivation, finance, absenceData,
|
||||
}: SecondarySchoolDetailViewProps) {
|
||||
const router = useRouter();
|
||||
// Hero map — the "View on map" link opens its fullscreen view.
|
||||
@@ -98,9 +99,9 @@ export function SecondarySchoolDetailView({
|
||||
|
||||
const secondaryAvg = nationalAvg?.secondary ?? {};
|
||||
|
||||
// GIAS OfficialSixthForm flag; missing (pipeline not yet re-run) => false.
|
||||
const hasSixthForm = schoolInfo.has_sixth_form ?? false;
|
||||
const hasSixthForm = schoolInfo.age_range?.includes('18') ?? false;
|
||||
const hasFinance = finance != null && finance.per_pupil_spend != null;
|
||||
const hasParents = parentView != null && parentView.total_responses != null && parentView.total_responses > 0;
|
||||
const hasDeprivation = deprivation != null && deprivation.idaci_decile != null;
|
||||
const hasLocation = schoolInfo.latitude != null && schoolInfo.longitude != null;
|
||||
const hasWellbeing = (latestResults?.sen_support_pct != null || latestResults?.sen_ehcp_pct != null) || hasDeprivation;
|
||||
@@ -158,6 +159,7 @@ export function SecondarySchoolDetailView({
|
||||
if (hasResults) navItems.push({ id: 'gcse', label: 'GCSEs' });
|
||||
if (admissions) navItems.push({ id: 'admissions', label: 'Admissions' });
|
||||
if (yearlyData.length > 1) navItems.push({ id: 'history', label: 'History' });
|
||||
if (hasParents) navItems.push({ id: 'parents', label: 'Parents' });
|
||||
if (hasWellbeing) navItems.push({ id: 'wellbeing', label: 'Wellbeing' });
|
||||
if (hasFinance) navItems.push({ id: 'finances', label: 'Finances' });
|
||||
|
||||
@@ -237,12 +239,6 @@ export function SecondarySchoolDetailView({
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{isProposedToClose(schoolInfo) && (
|
||||
<div className={styles.closingStrip} role="note">
|
||||
<strong>⚠ Proposed to close</strong> — this school is proposed for closure,
|
||||
check with the local authority before applying.
|
||||
</div>
|
||||
)}
|
||||
{schoolInfo.address && (
|
||||
<p className={styles.address}>
|
||||
{schoolInfo.address}{schoolInfo.postcode && `, ${schoolInfo.postcode}`}
|
||||
@@ -439,6 +435,11 @@ export function SecondarySchoolDetailView({
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{hasParents && (
|
||||
<p className={styles.parentRecommendLine}>
|
||||
<strong>{Math.round(parentView!.q_recommend_pct!)}%</strong> of parents would recommend this school ({parentView!.total_responses!.toLocaleString()} responses)
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
)}
|
||||
|
||||
@@ -774,6 +775,42 @@ export function SecondarySchoolDetailView({
|
||||
</details>
|
||||
</section>
|
||||
)}
|
||||
{/* ── Parent View ────────────────────────────────── */}
|
||||
{hasParents && parentView && (
|
||||
<section id="parents" className={styles.card}>
|
||||
<h2 className={styles.sectionTitle}>
|
||||
What Parents Say
|
||||
<span className={styles.responseBadge}>
|
||||
{parentView.total_responses!.toLocaleString()} responses
|
||||
</span>
|
||||
</h2>
|
||||
<p className={styles.sectionSubtitle}>
|
||||
From the Ofsted Parent View survey — parents share their experience of this school.
|
||||
</p>
|
||||
<div className={styles.parentViewGrid}>
|
||||
{[
|
||||
{ label: 'Would recommend this school', pct: parentView.q_recommend_pct },
|
||||
{ label: 'My child is happy here', pct: parentView.q_happy_pct },
|
||||
{ label: 'My child feels safe here', pct: parentView.q_safe_pct },
|
||||
{ label: 'Teaching is good', pct: parentView.q_teaching_pct },
|
||||
{ label: 'My child makes good progress', pct: parentView.q_progress_pct },
|
||||
{ label: 'School looks after pupils\' wellbeing', pct: parentView.q_wellbeing_pct },
|
||||
{ label: 'Behaviour is well managed', pct: parentView.q_behaviour_pct },
|
||||
{ label: 'School deals well with bullying', pct: parentView.q_bullying_pct },
|
||||
{ label: 'Communicates well with parents', pct: parentView.q_communication_pct },
|
||||
].filter(q => q.pct != null).map(({ label, pct }) => (
|
||||
<div key={label} className={styles.parentViewRow}>
|
||||
<span className={styles.parentViewLabel}>{label}</span>
|
||||
<div className={styles.parentViewBar}>
|
||||
<div className={styles.parentViewFill} style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
<span className={styles.parentViewPct}>{Math.round(pct!)}%</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{/* ── Wellbeing ──────────────────────────────────── */}
|
||||
{hasWellbeing && (
|
||||
<section id="wellbeing" className={styles.card}>
|
||||
|
||||
@@ -266,9 +266,3 @@
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
|
||||
.closingTag {
|
||||
background: #fdf6e3;
|
||||
color: #8a6200;
|
||||
border: 1px solid #e2c96f;
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
'use client';
|
||||
|
||||
import type { School } from '@/lib/types';
|
||||
import { buildOfstedListBadge, getPhaseStyle, schoolUrl, formatAgeRange, isProposedToClose } from '@/lib/utils';
|
||||
import { buildOfstedListBadge, getPhaseStyle, schoolUrl, formatAgeRange } from '@/lib/utils';
|
||||
import styles from './SecondarySchoolRow.module.css';
|
||||
|
||||
function detectAdmissionsTag(school: School): string | null {
|
||||
@@ -23,8 +23,7 @@ function detectAdmissionsTag(school: School): string | null {
|
||||
}
|
||||
|
||||
function hasSixthForm(school: School): boolean {
|
||||
// GIAS OfficialSixthForm flag; missing (pipeline not yet re-run) => false.
|
||||
return school.has_sixth_form ?? false;
|
||||
return school.age_range?.includes('18') ?? false;
|
||||
}
|
||||
|
||||
interface SecondarySchoolRowProps {
|
||||
@@ -97,9 +96,6 @@ export function SecondarySchoolRow({
|
||||
{admissionsTag}
|
||||
</span>
|
||||
)}
|
||||
{isProposedToClose(school) && (
|
||||
<span className={`${styles.provisionTag} ${styles.closingTag}`}>⚠ Proposed to close</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Line 3: KS4 stats */}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
/**
|
||||
* Viewport hook shared by the chart components.
|
||||
* Hydration-safe: SSR and the first client render report desktop; the
|
||||
* media-query subscription flips the value after mount.
|
||||
*/
|
||||
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
export function useIsMobile(maxWidth = 640): boolean {
|
||||
const [isMobile, setIsMobile] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const mq = window.matchMedia(`(max-width: ${maxWidth}px)`);
|
||||
const update = () => setIsMobile(mq.matches);
|
||||
update();
|
||||
mq.addEventListener('change', update);
|
||||
return () => mq.removeEventListener('change', update);
|
||||
}, [maxWidth]);
|
||||
|
||||
return isMobile;
|
||||
}
|
||||
@@ -29,7 +29,6 @@ export type EventName =
|
||||
| 'compare_viewed'
|
||||
| 'compare_metric_changed'
|
||||
| 'compare_shared'
|
||||
| 'compare_focus_school'
|
||||
// Operational
|
||||
| 'api_error'
|
||||
| 'results_load_more';
|
||||
|
||||
+20
-2
@@ -17,8 +17,6 @@ export interface School {
|
||||
school_type_code: string | null;
|
||||
religious_denomination: string | null;
|
||||
age_range: string | null;
|
||||
has_sixth_form?: boolean | null;
|
||||
status?: string | null; // GIAS establishment status ("Open" / "Open, but proposed to close")
|
||||
|
||||
// Address
|
||||
address1: string | null;
|
||||
@@ -101,6 +99,25 @@ export interface OfstedInspection {
|
||||
rc_sixth_form: number | null;
|
||||
}
|
||||
|
||||
export interface OfstedParentView {
|
||||
survey_date: string | null;
|
||||
total_responses: number | null;
|
||||
q_happy_pct: number | null;
|
||||
q_safe_pct: number | null;
|
||||
q_behaviour_pct: number | null;
|
||||
q_bullying_pct: number | null;
|
||||
q_communication_pct: number | null;
|
||||
q_progress_pct: number | null;
|
||||
q_teaching_pct: number | null;
|
||||
q_information_pct: number | null;
|
||||
q_curriculum_pct: number | null;
|
||||
q_future_pct: number | null;
|
||||
q_leadership_pct: number | null;
|
||||
q_wellbeing_pct: number | null;
|
||||
q_recommend_pct: number | null;
|
||||
q_sen_pct: number | null;
|
||||
}
|
||||
|
||||
export interface SchoolCensus {
|
||||
year: number;
|
||||
total_pupils: number | null;
|
||||
@@ -295,6 +312,7 @@ export interface SchoolDetailsResponse {
|
||||
absence_data: AbsenceData | null;
|
||||
// Supplementary data (null until Kestra populates)
|
||||
ofsted: OfstedInspection | null;
|
||||
parent_view: OfstedParentView | null;
|
||||
census: SchoolCensus | null;
|
||||
admissions: SchoolAdmissions | null;
|
||||
/** All available admissions years, oldest first. Drives the multi-year trend view. */
|
||||
|
||||
@@ -317,57 +317,6 @@ export function getTrendColor(trend: 'up' | 'down' | 'stable'): string {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Broad shape of a KS2/KS4 metric, used to scale chart axes and format values.
|
||||
*/
|
||||
export type MetricKind = 'percentage' | 'progress' | 'score';
|
||||
|
||||
export function metricKind(metric: string): MetricKind {
|
||||
if (metric.includes('progress')) return 'progress';
|
||||
if (metric.includes('pct') || metric.includes('rate')) return 'percentage';
|
||||
return 'score';
|
||||
}
|
||||
|
||||
/**
|
||||
* Fit a chart y-axis to the data instead of a fixed frame, so clustered
|
||||
* series remain distinguishable. Padding keeps a minimum span so noise is
|
||||
* not magnified into drama.
|
||||
*
|
||||
* - percentage: pad and snap to 5s; cap at 100; floor at 0 only when the
|
||||
* data is non-negative (some trend metrics have `pct` in the key but hold
|
||||
* negative year-over-year deltas).
|
||||
* - progress: symmetric around 0 so the zero line always shows.
|
||||
* - score (Attainment 8, scaled scores): pad and snap to integers; floor at
|
||||
* 0 only when the data is non-negative.
|
||||
*/
|
||||
export function computeYBounds(
|
||||
values: Array<number | null | undefined>,
|
||||
kind: MetricKind,
|
||||
): { min?: number; max?: number } {
|
||||
const nums = values.filter((v): v is number => typeof v === 'number' && Number.isFinite(v));
|
||||
if (nums.length === 0) return {};
|
||||
|
||||
const lo = Math.min(...nums);
|
||||
const hi = Math.max(...nums);
|
||||
|
||||
if (kind === 'progress') {
|
||||
const reach = Math.max(2, Math.ceil(Math.max(Math.abs(lo), Math.abs(hi)) + 0.5));
|
||||
return { min: -reach, max: reach };
|
||||
}
|
||||
|
||||
if (kind === 'percentage') {
|
||||
const pad = Math.max(5, Math.round((hi - lo) * 0.2));
|
||||
const min = Math.floor((lo - pad) / 5) * 5;
|
||||
const max = Math.min(100, Math.ceil((hi + pad) / 5) * 5);
|
||||
return { min: lo >= 0 ? Math.max(0, min) : min, max };
|
||||
}
|
||||
|
||||
// score
|
||||
const pad = Math.max(2, (hi - lo) * 0.2);
|
||||
const min = Math.floor(lo - pad);
|
||||
return { min: lo >= 0 ? Math.max(0, min) : min, max: Math.ceil(hi + pad) };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Local Storage Utilities
|
||||
// ============================================================================
|
||||
@@ -718,18 +667,3 @@ export function buildOfstedListBadge(school: {
|
||||
|
||||
return { label: 'Not yet inspected', cssClass: 'ofstedPending' };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Establishment status
|
||||
// ============================================================================
|
||||
|
||||
export const PROPOSED_TO_CLOSE_STATUS = 'Open, but proposed to close';
|
||||
|
||||
/**
|
||||
* GIAS lists some operating schools as "Open, but proposed to close".
|
||||
* They remain open (and may stay open if the proposal is withdrawn), but the
|
||||
* UI marks them so families check with the local authority before applying.
|
||||
*/
|
||||
export function isProposedToClose(school: { status?: string | null }): boolean {
|
||||
return school.status === PROPOSED_TO_CLOSE_STATUS;
|
||||
}
|
||||
|
||||
@@ -3,10 +3,21 @@ const nextConfig = {
|
||||
// Enable standalone output for Docker
|
||||
output: 'standalone',
|
||||
|
||||
// The /api/* and /sitemap.xml proxies to the FastAPI backend are route
|
||||
// handlers (app/api/[...path]/route.ts, app/sitemap.xml/route.ts) rather
|
||||
// than rewrites, so the backend host is read from FASTAPI_URL at runtime
|
||||
// instead of being baked into the build.
|
||||
// API Proxy to FastAPI backend
|
||||
async rewrites() {
|
||||
const apiUrl = process.env.FASTAPI_URL || 'http://localhost:8000/api';
|
||||
const backendUrl = apiUrl.replace(/\/api$/, '');
|
||||
return [
|
||||
{
|
||||
source: '/api/:path*',
|
||||
destination: `${apiUrl}/:path*`,
|
||||
},
|
||||
{
|
||||
source: '/sitemap.xml',
|
||||
destination: `${backendUrl}/sitemap.xml`,
|
||||
},
|
||||
];
|
||||
},
|
||||
|
||||
// Image optimization
|
||||
images: {
|
||||
|
||||
@@ -19,6 +19,7 @@ RUN pip install --no-cache-dir \
|
||||
./plugins/extractors/tap-uk-gias \
|
||||
./plugins/extractors/tap-uk-ees \
|
||||
./plugins/extractors/tap-uk-ofsted \
|
||||
./plugins/extractors/tap-uk-parent-view \
|
||||
./plugins/extractors/tap-uk-fbit \
|
||||
./plugins/extractors/tap-uk-idaci
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ print(f'Validation passed: {{count}} GIAS rows')
|
||||
|
||||
dbt_build = BashOperator(
|
||||
task_id="dbt_build",
|
||||
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_gias_establishments+ stg_gias_links+ gias_code_names+ --exclude int_ks2_with_lineage+ int_ks4_with_lineage+",
|
||||
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_gias_establishments+ stg_gias_links+ --exclude int_ks2_with_lineage+ int_ks4_with_lineage+",
|
||||
)
|
||||
|
||||
sync_typesense = BashOperator(
|
||||
@@ -156,6 +156,31 @@ with DAG(
|
||||
extract_ees_group >> dbt_build_ees >> sync_typesense_ees
|
||||
|
||||
|
||||
# ── Monthly DAG (Parent View) ──────────────────────────────────────────
|
||||
|
||||
with DAG(
|
||||
dag_id="school_data_monthly_parent_view",
|
||||
default_args=default_args,
|
||||
description="Monthly Ofsted Parent View extraction and transform",
|
||||
schedule="0 3 1 * *",
|
||||
start_date=datetime(2025, 1, 1),
|
||||
catchup=False,
|
||||
tags=["school-compare", "monthly"],
|
||||
) as monthly_parent_view_dag:
|
||||
|
||||
extract_parent_view = BashOperator(
|
||||
task_id="extract_parent_view",
|
||||
bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-parent-view target-postgres",
|
||||
)
|
||||
|
||||
dbt_build_parent_view = BashOperator(
|
||||
task_id="dbt_build",
|
||||
bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_parent_view+ fact_parent_view+",
|
||||
)
|
||||
|
||||
extract_parent_view >> dbt_build_parent_view
|
||||
|
||||
|
||||
# ── Annual DAG (IDACI Deprivation) ────────────────────────────────────
|
||||
|
||||
with DAG(
|
||||
|
||||
@@ -50,6 +50,11 @@ plugins:
|
||||
kind: string
|
||||
description: Ofsted Management Information download URL
|
||||
|
||||
- name: tap-uk-parent-view
|
||||
namespace: uk_parent_view
|
||||
pip_url: ./plugins/extractors/tap-uk-parent-view
|
||||
executable: tap-uk-parent-view
|
||||
|
||||
- name: tap-uk-fbit
|
||||
namespace: uk_fbit
|
||||
pip_url: ./plugins/extractors/tap-uk-fbit
|
||||
|
||||
@@ -31,22 +31,15 @@ class GIASEstablishmentsStream(Stream):
|
||||
schema = th.PropertiesList(
|
||||
th.Property("URN", th.IntegerType, required=True),
|
||||
th.Property("EstablishmentName", th.StringType),
|
||||
th.Property("TypeOfEstablishment (code)", th.StringType),
|
||||
th.Property("TypeOfEstablishment (name)", th.StringType),
|
||||
th.Property("PhaseOfEducation (code)", th.StringType),
|
||||
th.Property("PhaseOfEducation (name)", th.StringType),
|
||||
th.Property("OfficialSixthForm (code)", th.StringType),
|
||||
th.Property("OfficialSixthForm (name)", th.StringType),
|
||||
th.Property("LA (code)", th.StringType),
|
||||
th.Property("LA (name)", th.StringType),
|
||||
th.Property("EstablishmentNumber", th.StringType),
|
||||
th.Property("EstablishmentStatus (code)", th.StringType),
|
||||
th.Property("EstablishmentStatus (name)", th.StringType),
|
||||
th.Property("Postcode", th.StringType),
|
||||
th.Property("Gender (name)", th.StringType),
|
||||
th.Property("ReligiousCharacter (code)", th.StringType),
|
||||
th.Property("ReligiousCharacter (name)", th.StringType),
|
||||
th.Property("AdmissionsPolicy (code)", th.StringType),
|
||||
th.Property("AdmissionsPolicy (name)", th.StringType),
|
||||
th.Property("SchoolCapacity", th.StringType),
|
||||
th.Property("NumberOfPupils", th.StringType),
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "tap-uk-parent-view"
|
||||
version = "0.1.0"
|
||||
description = "Singer tap for UK Ofsted Parent View survey data"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"singer-sdk~=0.53",
|
||||
"requests>=2.31",
|
||||
"pandas>=2.0",
|
||||
"openpyxl>=3.1",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
tap-uk-parent-view = "tap_uk_parent_view.tap:TapUKParentView.cli"
|
||||
@@ -0,0 +1 @@
|
||||
"""tap-uk-parent-view: Singer tap for Ofsted Parent View survey data."""
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Parent View Singer tap — extracts survey data from Ofsted Parent View open data portal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from singer_sdk import Stream, Tap
|
||||
from singer_sdk import typing as th
|
||||
|
||||
OPEN_DATA_PAGE = "https://parentview.ofsted.gov.uk/open-data"
|
||||
|
||||
|
||||
def _positive_pct(row: pd.Series, q_col_base: str) -> float | None:
|
||||
"""Sum 'Strongly agree' + 'Agree' percentages for a question."""
|
||||
strongly = row.get(f"{q_col_base} - Strongly agree %") or row.get(f"{q_col_base} - Strongly Agree %")
|
||||
agree = row.get(f"{q_col_base} - Agree %")
|
||||
try:
|
||||
total = 0.0
|
||||
if pd.notna(strongly):
|
||||
total += float(strongly)
|
||||
if pd.notna(agree):
|
||||
total += float(agree)
|
||||
return round(total, 1) if total > 0 else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class ParentViewStream(Stream):
|
||||
"""Stream: Parent View survey responses per school."""
|
||||
|
||||
name = "parent_view"
|
||||
primary_keys = ["urn"]
|
||||
replication_key = None
|
||||
|
||||
schema = th.PropertiesList(
|
||||
th.Property("urn", th.IntegerType, required=True),
|
||||
th.Property("survey_date", th.StringType),
|
||||
th.Property("total_responses", th.IntegerType),
|
||||
th.Property("q_happy_pct", th.NumberType),
|
||||
th.Property("q_safe_pct", th.NumberType),
|
||||
th.Property("q_behaviour_pct", th.NumberType),
|
||||
th.Property("q_bullying_pct", th.NumberType),
|
||||
th.Property("q_communication_pct", th.NumberType),
|
||||
th.Property("q_progress_pct", th.NumberType),
|
||||
th.Property("q_teaching_pct", th.NumberType),
|
||||
th.Property("q_information_pct", th.NumberType),
|
||||
th.Property("q_curriculum_pct", th.NumberType),
|
||||
th.Property("q_future_pct", th.NumberType),
|
||||
th.Property("q_leadership_pct", th.NumberType),
|
||||
th.Property("q_wellbeing_pct", th.NumberType),
|
||||
th.Property("q_recommend_pct", th.NumberType),
|
||||
).to_dict()
|
||||
|
||||
def _discover_download_url(self) -> str:
|
||||
"""Scrape the open data page for the download link."""
|
||||
resp = requests.get(OPEN_DATA_PAGE, timeout=30)
|
||||
resp.raise_for_status()
|
||||
urls = re.findall(r'href="([^"]+\.(?:xlsx|csv|zip))"', resp.text, re.IGNORECASE)
|
||||
if not urls:
|
||||
msg = "No download link found on Parent View open data page"
|
||||
raise RuntimeError(msg)
|
||||
url = urls[0]
|
||||
if not url.startswith("http"):
|
||||
url = "https://parentview.ofsted.gov.uk" + url
|
||||
return url
|
||||
|
||||
def get_records(self, context):
|
||||
url = self._discover_download_url()
|
||||
self.logger.info("Downloading Parent View data: %s", url)
|
||||
|
||||
resp = requests.get(url, timeout=120)
|
||||
resp.raise_for_status()
|
||||
|
||||
if url.endswith(".xlsx"):
|
||||
df = pd.read_excel(io.BytesIO(resp.content))
|
||||
else:
|
||||
df = pd.read_csv(
|
||||
io.BytesIO(resp.content),
|
||||
encoding="latin-1",
|
||||
low_memory=False,
|
||||
)
|
||||
|
||||
# Normalise URN column
|
||||
urn_col = next((c for c in df.columns if c.strip().upper() == "URN"), None)
|
||||
if not urn_col:
|
||||
self.logger.error("URN column not found. Columns: %s", list(df.columns)[:20])
|
||||
return
|
||||
|
||||
df.rename(columns={urn_col: "urn"}, inplace=True)
|
||||
df["urn"] = pd.to_numeric(df["urn"], errors="coerce")
|
||||
df = df.dropna(subset=["urn"])
|
||||
|
||||
# Find total responses column
|
||||
resp_col = next(
|
||||
(c for c in df.columns if "total" in c.lower() and "respon" in c.lower()),
|
||||
None,
|
||||
)
|
||||
|
||||
today = date.today().isoformat()
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
urn = int(row["urn"])
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
total = None
|
||||
if resp_col and pd.notna(row.get(resp_col)):
|
||||
try:
|
||||
total = int(row[resp_col])
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
yield {
|
||||
"urn": urn,
|
||||
"survey_date": today,
|
||||
"total_responses": total,
|
||||
"q_happy_pct": _positive_pct(row, "Q1"),
|
||||
"q_safe_pct": _positive_pct(row, "Q2"),
|
||||
"q_behaviour_pct": _positive_pct(row, "Q3"),
|
||||
"q_bullying_pct": _positive_pct(row, "Q4"),
|
||||
"q_communication_pct": _positive_pct(row, "Q5"),
|
||||
"q_progress_pct": _positive_pct(row, "Q7"),
|
||||
"q_teaching_pct": _positive_pct(row, "Q8"),
|
||||
"q_information_pct": _positive_pct(row, "Q9"),
|
||||
"q_curriculum_pct": _positive_pct(row, "Q10"),
|
||||
"q_future_pct": _positive_pct(row, "Q11"),
|
||||
"q_leadership_pct": _positive_pct(row, "Q12"),
|
||||
"q_wellbeing_pct": _positive_pct(row, "Q13"),
|
||||
"q_recommend_pct": _positive_pct(row, "Q14"),
|
||||
}
|
||||
|
||||
|
||||
class TapUKParentView(Tap):
|
||||
"""Singer tap for UK Ofsted Parent View."""
|
||||
|
||||
name = "tap-uk-parent-view"
|
||||
config_jsonschema = th.PropertiesList(
|
||||
th.Property("download_url", th.StringType, description="Direct URL to Parent View data file"),
|
||||
).to_dict()
|
||||
|
||||
def discover_streams(self):
|
||||
return [ParentViewStream(self)]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
TapUKParentView.cli()
|
||||
@@ -1,134 +0,0 @@
|
||||
"""Generate GIAS code->name dictionaries from the live bulk CSV.
|
||||
|
||||
Writes:
|
||||
- backend/gias_codes.py (canonical Python module)
|
||||
- pipeline/scripts/gias_codes.py (byte-identical copy)
|
||||
- pipeline/transform/seeds/gias_code_names.csv (dbt seed for drift test)
|
||||
|
||||
Run from the repo root whenever the dbt drift test warns that DfE
|
||||
added/renamed a value: python pipeline/scripts/generate_gias_codes.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
GIAS_URL = (
|
||||
"https://ea-edubase-api-prod.azurewebsites.net"
|
||||
"/edubase/downloads/public/edubasealldata{date}.csv"
|
||||
)
|
||||
|
||||
# (CSV code column, CSV name column, python dict name, seed field key)
|
||||
FIELDS = [
|
||||
("TypeOfEstablishment (code)", "TypeOfEstablishment (name)", "SCHOOL_TYPE", "school_type"),
|
||||
("EstablishmentStatus (code)", "EstablishmentStatus (name)", "ESTABLISHMENT_STATUS", "establishment_status"),
|
||||
("PhaseOfEducation (code)", "PhaseOfEducation (name)", "PHASE_OF_EDUCATION", "phase_of_education"),
|
||||
("OfficialSixthForm (code)", "OfficialSixthForm (name)", "OFFICIAL_SIXTH_FORM", "official_sixth_form"),
|
||||
("ReligiousCharacter (code)", "ReligiousCharacter (name)", "RELIGIOUS_CHARACTER", "religious_character"),
|
||||
("AdmissionsPolicy (code)", "AdmissionsPolicy (name)", "ADMISSIONS_POLICY", "admissions_policy"),
|
||||
]
|
||||
|
||||
MODULE_HEADER = '''"""GIAS code -> name dictionaries.
|
||||
|
||||
GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
|
||||
— do not edit by hand; rerun the script when the dbt drift test warns.
|
||||
The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
|
||||
must be byte-identical (enforced by backend/tests/test_gias_codes.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
'''
|
||||
|
||||
MODULE_FOOTER = '''
|
||||
|
||||
def translate(code, mapping: dict[int, str]) -> str | None:
|
||||
"""Translate a GIAS code to its display name.
|
||||
|
||||
None/NaN -> None (column absent or suppressed). Unknown codes degrade to
|
||||
"Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
|
||||
"""
|
||||
if code is None or (isinstance(code, float) and math.isnan(code)):
|
||||
return None
|
||||
code = int(code)
|
||||
if code not in mapping:
|
||||
logger.warning("Unknown GIAS code %s (not in dictionary)", code)
|
||||
return f"Unknown ({code})"
|
||||
return mapping[code]
|
||||
'''
|
||||
|
||||
|
||||
def download_csv() -> pd.DataFrame:
|
||||
for day in (date.today(), date.today() - timedelta(days=1)):
|
||||
url = GIAS_URL.format(date=day.strftime("%Y%m%d"))
|
||||
print(f"Downloading {url}")
|
||||
resp = requests.get(url, timeout=300)
|
||||
if resp.status_code == 404:
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return pd.read_csv(
|
||||
io.StringIO(resp.content.decode("latin-1")),
|
||||
dtype=str, keep_default_na=False,
|
||||
)
|
||||
sys.exit("GIAS CSV not available for today or yesterday")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
repo = Path(__file__).resolve().parents[2]
|
||||
df = download_csv()
|
||||
|
||||
module_parts = [MODULE_HEADER]
|
||||
seed_rows: list[tuple[str, int, str]] = []
|
||||
|
||||
for code_col, name_col, dict_name, field_key in FIELDS:
|
||||
pairs = (
|
||||
df[[code_col, name_col]]
|
||||
.loc[lambda d: (d[code_col] != "") & (d[name_col] != "")]
|
||||
.drop_duplicates()
|
||||
)
|
||||
mapping = sorted((int(c), n) for c, n in pairs.itertuples(index=False))
|
||||
dupes = len(mapping) - len({c for c, _ in mapping})
|
||||
if dupes:
|
||||
sys.exit(f"{code_col}: {dupes} codes map to multiple names — investigate before generating")
|
||||
lines = [f"{dict_name}: dict[int, str] = {{"]
|
||||
for code, name in mapping:
|
||||
escaped = name.replace('"', '\\"')
|
||||
lines.append(f' {code}: "{escaped}",')
|
||||
lines.append("}\n")
|
||||
module_parts.append("\n".join(lines))
|
||||
seed_rows += [(field_key, code, name) for code, name in mapping]
|
||||
|
||||
module = "\n".join(module_parts) + MODULE_FOOTER
|
||||
|
||||
(repo / "backend" / "gias_codes.py").write_text(module)
|
||||
(repo / "pipeline" / "scripts" / "gias_codes.py").write_text(module)
|
||||
|
||||
seed_path = repo / "pipeline" / "transform" / "seeds" / "gias_code_names.csv"
|
||||
with open(seed_path, "w", newline="") as fh:
|
||||
import csv
|
||||
w = csv.writer(fh)
|
||||
w.writerow(["field", "code", "name"])
|
||||
w.writerows(seed_rows)
|
||||
|
||||
print(f"Wrote backend/gias_codes.py, pipeline/scripts/gias_codes.py, {seed_path.name}")
|
||||
print("\nKey codes for the dbt work (Task 3):")
|
||||
for field in ("establishment_status", "phase_of_education", "official_sixth_form"):
|
||||
print(f" {field}:")
|
||||
for f, code, name in seed_rows:
|
||||
if f == field:
|
||||
print(f" {code} = {name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,152 +0,0 @@
|
||||
"""GIAS code -> name dictionaries.
|
||||
|
||||
GENERATED by pipeline/scripts/generate_gias_codes.py from the GIAS bulk CSV
|
||||
— do not edit by hand; rerun the script when the dbt drift test warns.
|
||||
The canonical file is backend/gias_codes.py; pipeline/scripts/gias_codes.py
|
||||
must be byte-identical (enforced by backend/tests/test_gias_codes.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SCHOOL_TYPE: dict[int, str] = {
|
||||
1: "Community school",
|
||||
2: "Voluntary aided school",
|
||||
3: "Voluntary controlled school",
|
||||
5: "Foundation school",
|
||||
6: "City technology college",
|
||||
7: "Community special school",
|
||||
8: "Non-maintained special school",
|
||||
10: "Other independent special school",
|
||||
11: "Other independent school",
|
||||
12: "Foundation special school",
|
||||
14: "Pupil referral unit",
|
||||
15: "Local authority nursery school",
|
||||
18: "Further education",
|
||||
24: "Secure units",
|
||||
25: "Offshore schools",
|
||||
26: "Service children's education",
|
||||
27: "Miscellaneous",
|
||||
28: "Academy sponsor led",
|
||||
29: "Higher education institutions",
|
||||
30: "Welsh establishment",
|
||||
31: "Sixth form centres",
|
||||
32: "Special post 16 institution",
|
||||
33: "Academy special sponsor led",
|
||||
34: "Academy converter",
|
||||
35: "Free schools",
|
||||
36: "Free schools special",
|
||||
37: "British schools overseas",
|
||||
38: "Free schools alternative provision",
|
||||
39: "Free schools 16 to 19",
|
||||
40: "University technical college",
|
||||
41: "Studio schools",
|
||||
42: "Academy alternative provision converter",
|
||||
43: "Academy alternative provision sponsor led",
|
||||
44: "Academy special converter",
|
||||
45: "Academy 16-19 converter",
|
||||
46: "Academy 16 to 19 sponsor led",
|
||||
49: "Online provider",
|
||||
56: "Institution funded by other government department",
|
||||
57: "Academy secure 16 to 19",
|
||||
}
|
||||
|
||||
ESTABLISHMENT_STATUS: dict[int, str] = {
|
||||
1: "Open",
|
||||
2: "Closed",
|
||||
3: "Open, but proposed to close",
|
||||
4: "Proposed to open",
|
||||
}
|
||||
|
||||
PHASE_OF_EDUCATION: dict[int, str] = {
|
||||
0: "Not applicable",
|
||||
1: "Nursery",
|
||||
2: "Primary",
|
||||
3: "Middle deemed primary",
|
||||
4: "Secondary",
|
||||
5: "Middle deemed secondary",
|
||||
6: "16 plus",
|
||||
7: "All-through",
|
||||
}
|
||||
|
||||
OFFICIAL_SIXTH_FORM: dict[int, str] = {
|
||||
0: "Not applicable",
|
||||
1: "Has a sixth form",
|
||||
2: "Does not have a sixth form",
|
||||
}
|
||||
|
||||
RELIGIOUS_CHARACTER: dict[int, str] = {
|
||||
0: "Does not apply",
|
||||
2: "Church of England",
|
||||
3: "Roman Catholic",
|
||||
4: "Methodist",
|
||||
5: "Jewish",
|
||||
6: "None",
|
||||
7: "Muslim",
|
||||
8: "Seventh Day Adventist",
|
||||
9: "Church of England/Methodist",
|
||||
10: "Methodist/Church of England",
|
||||
11: "Church of England/Roman Catholic",
|
||||
12: "Church of England/United Reformed Church",
|
||||
13: "Roman Catholic/Church of England",
|
||||
14: "Quaker",
|
||||
15: "Christian",
|
||||
16: "United Reformed Church",
|
||||
17: "Congregational Church",
|
||||
18: "Free Church",
|
||||
19: "Church of England/Free Church",
|
||||
20: "Church of England/Christian",
|
||||
21: "Sikh",
|
||||
22: "Greek Orthodox",
|
||||
24: "Buddhist",
|
||||
25: "Hindu",
|
||||
26: "Moravian",
|
||||
28: "Inter- / non- denominational",
|
||||
29: "Multi-faith",
|
||||
30: "Church of England/Methodist/United Reform Church/Baptist",
|
||||
31: "Anglican",
|
||||
32: "Anglican/Christian",
|
||||
33: "Anglican/Evangelical",
|
||||
34: "Anglican/Church of England",
|
||||
35: "Catholic",
|
||||
36: "Charadi Jewish",
|
||||
37: "Christian/Evangelical",
|
||||
38: "Christian Science",
|
||||
39: "Christian/Methodist",
|
||||
40: "Christian/non-denominational",
|
||||
41: "Church of England/Evangelical",
|
||||
42: "Islam",
|
||||
43: "Orthodox Jewish",
|
||||
44: "Plymouth Brethren Christian Church",
|
||||
45: "Protestant",
|
||||
46: "Protestant/Evangelical",
|
||||
47: "Reformed Baptist",
|
||||
48: "Roman Catholic/Anglican",
|
||||
49: "Sunni Deobandi",
|
||||
}
|
||||
|
||||
ADMISSIONS_POLICY: dict[int, str] = {
|
||||
0: "Not applicable",
|
||||
2: "Selective",
|
||||
4: "Non-selective",
|
||||
}
|
||||
|
||||
|
||||
def translate(code, mapping: dict[int, str]) -> str | None:
|
||||
"""Translate a GIAS code to its display name.
|
||||
|
||||
None/NaN -> None (column absent or suppressed). Unknown codes degrade to
|
||||
"Unknown (<code>)" with a warning so a new DfE value never blanks the UI.
|
||||
"""
|
||||
if code is None or (isinstance(code, float) and math.isnan(code)):
|
||||
return None
|
||||
code = int(code)
|
||||
if code not in mapping:
|
||||
logger.warning("Unknown GIAS code %s (not in dictionary)", code)
|
||||
return f"Unknown ({code})"
|
||||
return mapping[code]
|
||||
@@ -19,8 +19,6 @@ import psycopg2
|
||||
import psycopg2.extras
|
||||
import typesense
|
||||
|
||||
from gias_codes import PHASE_OF_EDUCATION, RELIGIOUS_CHARACTER, SCHOOL_TYPE, translate
|
||||
|
||||
COLLECTION_SCHEMA = {
|
||||
"fields": [
|
||||
{"name": "urn", "type": "int32"},
|
||||
@@ -46,10 +44,10 @@ QUERY_BASE = """
|
||||
SELECT
|
||||
s.urn,
|
||||
s.school_name,
|
||||
s.phase_code,
|
||||
s.school_type_code,
|
||||
s.phase,
|
||||
s.school_type,
|
||||
l.local_authority_name as local_authority,
|
||||
s.religious_character_code,
|
||||
s.religious_character,
|
||||
s.ofsted_grade,
|
||||
l.postcode,
|
||||
s.headteacher_name,
|
||||
@@ -87,15 +85,14 @@ def build_document(row: dict) -> dict:
|
||||
"id": str(row["urn"]),
|
||||
"urn": row["urn"],
|
||||
"school_name": row["school_name"] or "",
|
||||
"phase": translate(row["phase_code"], PHASE_OF_EDUCATION) or "",
|
||||
"school_type": translate(row["school_type_code"], SCHOOL_TYPE) or "",
|
||||
"phase": row["phase"] or "",
|
||||
"school_type": row["school_type"] or "",
|
||||
"local_authority": row["local_authority"] or "",
|
||||
"postcode": row["postcode"] or "",
|
||||
}
|
||||
|
||||
religious_character = translate(row.get("religious_character_code"), RELIGIOUS_CHARACTER)
|
||||
if religious_character:
|
||||
doc["religious_character"] = religious_character
|
||||
if row.get("religious_character"):
|
||||
doc["religious_character"] = row["religious_character"]
|
||||
if row.get("ofsted_grade"):
|
||||
doc["ofsted_rating"] = OFSTED_LABELS.get(row["ofsted_grade"], "")
|
||||
if row.get("headteacher_name"):
|
||||
|
||||
@@ -8,46 +8,18 @@ models:
|
||||
tests: [not_null, unique]
|
||||
- name: school_name
|
||||
tests: [not_null]
|
||||
- name: phase_code
|
||||
- name: phase
|
||||
description: >
|
||||
GIAS PhaseOfEducation code (2 = Primary, 4 = Secondary, 7 = All-through,
|
||||
etc. — see seeds/gias_code_names.csv). May be null for a small number
|
||||
Primary / Secondary / All-through etc. May be null for a small number
|
||||
of independent schools where GIAS publishes "Not Applicable", no
|
||||
statutory age range, and the school name gives no hint.
|
||||
tests:
|
||||
- not_null:
|
||||
severity: warn
|
||||
- name: has_sixth_form
|
||||
description: >
|
||||
Authoritative sixth-form flag from GIAS OfficialSixthForm.
|
||||
"Has a sixth form" => true; "Does not have a sixth form" and
|
||||
"Not applicable" => false; blank GIAS value falls back to
|
||||
statutory_high_age >= 18. Replaces the age_range-contains-"18"
|
||||
heuristic (spec 2026-07-07 §3).
|
||||
tests:
|
||||
- not_null
|
||||
- accepted_values:
|
||||
values: [true, false]
|
||||
- name: status_code
|
||||
description: GIAS EstablishmentStatus code (1 = Open, 3 = Open but proposed to close)
|
||||
- name: status
|
||||
tests:
|
||||
- accepted_values:
|
||||
values: [1, 3]
|
||||
- name: school_type_code
|
||||
tests:
|
||||
- accepted_values:
|
||||
severity: warn
|
||||
values: [1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 14, 15, 18, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 49, 56, 57]
|
||||
- name: religious_character_code
|
||||
tests:
|
||||
- accepted_values:
|
||||
severity: warn
|
||||
values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
|
||||
- name: admissions_policy_code
|
||||
tests:
|
||||
- accepted_values:
|
||||
severity: warn
|
||||
values: [0, 2, 4]
|
||||
values: ["Open"]
|
||||
|
||||
- name: dim_location
|
||||
description: School location dimension with PostGIS geometry
|
||||
@@ -133,6 +105,12 @@ models:
|
||||
- name: year
|
||||
tests: [not_null]
|
||||
|
||||
- name: fact_parent_view
|
||||
description: Parent View survey responses
|
||||
columns:
|
||||
- name: urn
|
||||
tests: [not_null]
|
||||
|
||||
- name: fact_ks2_national_averages
|
||||
description: Official DfE KS2 national headline averages — one row per academic year
|
||||
columns:
|
||||
|
||||
@@ -31,5 +31,4 @@ select
|
||||
else null
|
||||
end as longitude
|
||||
from {{ ref('stg_gias_establishments') }} s
|
||||
-- Must match dim_school's status filter exactly (the API inner-joins the two).
|
||||
where s.status_code in (1, 3)
|
||||
where s.status = 'Open'
|
||||
|
||||
@@ -19,17 +19,16 @@ select
|
||||
s.urn,
|
||||
s.local_authority_code * 1000 + s.establishment_number as laestab,
|
||||
s.school_name,
|
||||
-- Phase in GIAS code space (see seeds/gias_code_names.csv):
|
||||
-- 2 = Primary, 4 = Secondary, 7 = All-through, 0 = Not applicable.
|
||||
case
|
||||
-- 1. Trust GIAS phase when it's a real value (0 = the catch-all "Not Applicable")
|
||||
when s.phase_code is not null and s.phase_code != 0
|
||||
then s.phase_code
|
||||
-- 1. Trust GIAS phase when it's a real value (not the catch-all "Not Applicable")
|
||||
when s.phase is not null
|
||||
and lower(trim(s.phase)) not in ('not applicable', '', 'unknown')
|
||||
then s.phase
|
||||
-- 2. Infer from statutory age range (independent schools still publish these)
|
||||
when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 2
|
||||
when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 4
|
||||
when s.statutory_high_age is not null and s.statutory_high_age <= 11 then 'Primary'
|
||||
when s.statutory_low_age is not null and s.statutory_low_age >= 11 then 'Secondary'
|
||||
when s.statutory_low_age is not null and s.statutory_high_age is not null
|
||||
and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 7
|
||||
and s.statutory_low_age < 11 and s.statutory_high_age > 11 then 'All-through'
|
||||
-- 3. Fallback: infer from school name (covers independents with missing ages)
|
||||
when s.school_name ilike '%primary%'
|
||||
or s.school_name ilike '%infant%'
|
||||
@@ -37,29 +36,22 @@ select
|
||||
or s.school_name ilike '%preparatory%'
|
||||
or s.school_name ilike '% prep school%'
|
||||
or s.school_name ilike '% prep %'
|
||||
then 2
|
||||
then 'Primary'
|
||||
when s.school_name ilike '%secondary%'
|
||||
or s.school_name ilike '%high school%'
|
||||
or s.school_name ilike '%grammar%'
|
||||
or s.school_name ilike '%senior school%'
|
||||
or s.school_name ilike '%upper school%'
|
||||
then 4
|
||||
-- 4. Give up — null renders no phase pill
|
||||
then 'Secondary'
|
||||
-- 4. Give up — leave phase null so the UI renders no pill
|
||||
else null
|
||||
end as phase_code,
|
||||
s.school_type_code,
|
||||
end as phase,
|
||||
s.school_type,
|
||||
s.academy_trust_name,
|
||||
s.academy_trust_uid,
|
||||
s.religious_character_code,
|
||||
s.religious_character,
|
||||
s.gender,
|
||||
s.statutory_low_age || '-' || s.statutory_high_age as age_range,
|
||||
-- GIAS OfficialSixthForm in code space: 1 = has, 2 = does not, 0 = N/A.
|
||||
-- Null (rare, new establishments) falls back to the statutory age range.
|
||||
case
|
||||
when s.official_sixth_form_code = 1 then true
|
||||
when s.official_sixth_form_code in (0, 2) then false
|
||||
else coalesce(s.statutory_high_age >= 18, false)
|
||||
end as has_sixth_form,
|
||||
s.capacity,
|
||||
s.total_pupils,
|
||||
concat_ws(' ', s.head_title, s.head_first_name, s.head_last_name) as headteacher_name,
|
||||
@@ -67,9 +59,9 @@ select
|
||||
s.telephone,
|
||||
s.open_date,
|
||||
s.close_date,
|
||||
s.status_code,
|
||||
s.status,
|
||||
s.nursery_provision,
|
||||
s.admissions_policy_code,
|
||||
s.admissions_policy,
|
||||
|
||||
-- Latest Ofsted (populated after monthly Ofsted pipeline runs)
|
||||
{% if ofsted_relation is not none %}
|
||||
@@ -88,6 +80,4 @@ from schools s
|
||||
{% if ofsted_relation is not none %}
|
||||
left join {{ ref('int_ofsted_latest') }} o on s.urn = o.urn
|
||||
{% endif %}
|
||||
-- 1 = Open; 3 = Open, but proposed to close (still operating; drops out when
|
||||
-- GIAS flips to Closed — marts fully rebuild each run).
|
||||
where s.status_code in (1, 3)
|
||||
where s.status = 'Open'
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
-- Mart: Parent View survey responses — one row per URN (latest survey)
|
||||
|
||||
select
|
||||
urn,
|
||||
survey_date,
|
||||
total_responses,
|
||||
q_happy_pct,
|
||||
q_safe_pct,
|
||||
q_behaviour_pct,
|
||||
q_bullying_pct,
|
||||
q_communication_pct,
|
||||
q_progress_pct,
|
||||
q_teaching_pct,
|
||||
q_information_pct,
|
||||
q_curriculum_pct,
|
||||
q_future_pct,
|
||||
q_leadership_pct,
|
||||
q_wellbeing_pct,
|
||||
q_recommend_pct
|
||||
from {{ ref('stg_parent_view') }}
|
||||
@@ -53,6 +53,9 @@ sources:
|
||||
|
||||
# Phonics: no school-level data on EES (only national/LA level)
|
||||
|
||||
- name: parent_view
|
||||
description: Ofsted Parent View survey responses
|
||||
|
||||
- name: fbit_finance
|
||||
description: Financial benchmarking data from FBIT API
|
||||
|
||||
|
||||
@@ -12,12 +12,11 @@ renamed as (
|
||||
"LA (name)" as local_authority_name,
|
||||
cast(nullif("EstablishmentNumber", '') as integer) as establishment_number,
|
||||
"EstablishmentName" as school_name,
|
||||
cast(nullif(trim("TypeOfEstablishment (code)"), '') as integer) as school_type_code,
|
||||
cast(nullif(trim("PhaseOfEducation (code)"), '') as integer) as phase_code,
|
||||
cast(nullif(trim("OfficialSixthForm (code)"), '') as integer) as official_sixth_form_code,
|
||||
"TypeOfEstablishment (name)" as school_type,
|
||||
"PhaseOfEducation (name)" as phase,
|
||||
"Gender (name)" as gender,
|
||||
cast(nullif(trim("ReligiousCharacter (code)"), '') as integer) as religious_character_code,
|
||||
cast(nullif(trim("AdmissionsPolicy (code)"), '') as integer) as admissions_policy_code,
|
||||
"ReligiousCharacter (name)" as religious_character,
|
||||
"AdmissionsPolicy (name)" as admissions_policy,
|
||||
"SchoolCapacity" as capacity,
|
||||
cast(nullif("NumberOfPupils", '') as integer) as total_pupils,
|
||||
"HeadTitle (name)" as head_title,
|
||||
@@ -30,7 +29,7 @@ renamed as (
|
||||
"Town" as town,
|
||||
"County (name)" as county,
|
||||
"Postcode" as postcode,
|
||||
cast(nullif(trim("EstablishmentStatus (code)"), '') as integer) as status_code,
|
||||
"EstablishmentStatus (name)" as status,
|
||||
case when "OpenDate" = '' then null else to_date("OpenDate", 'DD-MM-YYYY') end as open_date,
|
||||
case when "CloseDate" = '' then null else to_date("CloseDate", 'DD-MM-YYYY') end as close_date,
|
||||
"Trusts (name)" as academy_trust_name,
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
-- Staging model: Ofsted Parent View survey responses
|
||||
-- The tap computes positive percentages (Strongly agree + Agree) per question.
|
||||
|
||||
with source as (
|
||||
select * from {{ source('raw', 'parent_view') }}
|
||||
),
|
||||
|
||||
renamed as (
|
||||
select
|
||||
cast(urn as integer) as urn,
|
||||
cast(survey_date as date) as survey_date,
|
||||
cast(total_responses as integer) as total_responses,
|
||||
cast(q_happy_pct as numeric) as q_happy_pct,
|
||||
cast(q_safe_pct as numeric) as q_safe_pct,
|
||||
cast(q_behaviour_pct as numeric) as q_behaviour_pct,
|
||||
cast(q_bullying_pct as numeric) as q_bullying_pct,
|
||||
cast(q_communication_pct as numeric) as q_communication_pct,
|
||||
cast(q_progress_pct as numeric) as q_progress_pct,
|
||||
cast(q_teaching_pct as numeric) as q_teaching_pct,
|
||||
cast(q_information_pct as numeric) as q_information_pct,
|
||||
cast(q_curriculum_pct as numeric) as q_curriculum_pct,
|
||||
cast(q_future_pct as numeric) as q_future_pct,
|
||||
cast(q_leadership_pct as numeric) as q_leadership_pct,
|
||||
cast(q_wellbeing_pct as numeric) as q_wellbeing_pct,
|
||||
cast(q_recommend_pct as numeric) as q_recommend_pct
|
||||
from source
|
||||
where urn is not null
|
||||
)
|
||||
|
||||
select * from renamed
|
||||
@@ -1,105 +0,0 @@
|
||||
field,code,name
|
||||
school_type,1,Community school
|
||||
school_type,2,Voluntary aided school
|
||||
school_type,3,Voluntary controlled school
|
||||
school_type,5,Foundation school
|
||||
school_type,6,City technology college
|
||||
school_type,7,Community special school
|
||||
school_type,8,Non-maintained special school
|
||||
school_type,10,Other independent special school
|
||||
school_type,11,Other independent school
|
||||
school_type,12,Foundation special school
|
||||
school_type,14,Pupil referral unit
|
||||
school_type,15,Local authority nursery school
|
||||
school_type,18,Further education
|
||||
school_type,24,Secure units
|
||||
school_type,25,Offshore schools
|
||||
school_type,26,Service children's education
|
||||
school_type,27,Miscellaneous
|
||||
school_type,28,Academy sponsor led
|
||||
school_type,29,Higher education institutions
|
||||
school_type,30,Welsh establishment
|
||||
school_type,31,Sixth form centres
|
||||
school_type,32,Special post 16 institution
|
||||
school_type,33,Academy special sponsor led
|
||||
school_type,34,Academy converter
|
||||
school_type,35,Free schools
|
||||
school_type,36,Free schools special
|
||||
school_type,37,British schools overseas
|
||||
school_type,38,Free schools alternative provision
|
||||
school_type,39,Free schools 16 to 19
|
||||
school_type,40,University technical college
|
||||
school_type,41,Studio schools
|
||||
school_type,42,Academy alternative provision converter
|
||||
school_type,43,Academy alternative provision sponsor led
|
||||
school_type,44,Academy special converter
|
||||
school_type,45,Academy 16-19 converter
|
||||
school_type,46,Academy 16 to 19 sponsor led
|
||||
school_type,49,Online provider
|
||||
school_type,56,Institution funded by other government department
|
||||
school_type,57,Academy secure 16 to 19
|
||||
establishment_status,1,Open
|
||||
establishment_status,2,Closed
|
||||
establishment_status,3,"Open, but proposed to close"
|
||||
establishment_status,4,Proposed to open
|
||||
phase_of_education,0,Not applicable
|
||||
phase_of_education,1,Nursery
|
||||
phase_of_education,2,Primary
|
||||
phase_of_education,3,Middle deemed primary
|
||||
phase_of_education,4,Secondary
|
||||
phase_of_education,5,Middle deemed secondary
|
||||
phase_of_education,6,16 plus
|
||||
phase_of_education,7,All-through
|
||||
official_sixth_form,0,Not applicable
|
||||
official_sixth_form,1,Has a sixth form
|
||||
official_sixth_form,2,Does not have a sixth form
|
||||
religious_character,0,Does not apply
|
||||
religious_character,2,Church of England
|
||||
religious_character,3,Roman Catholic
|
||||
religious_character,4,Methodist
|
||||
religious_character,5,Jewish
|
||||
religious_character,6,None
|
||||
religious_character,7,Muslim
|
||||
religious_character,8,Seventh Day Adventist
|
||||
religious_character,9,Church of England/Methodist
|
||||
religious_character,10,Methodist/Church of England
|
||||
religious_character,11,Church of England/Roman Catholic
|
||||
religious_character,12,Church of England/United Reformed Church
|
||||
religious_character,13,Roman Catholic/Church of England
|
||||
religious_character,14,Quaker
|
||||
religious_character,15,Christian
|
||||
religious_character,16,United Reformed Church
|
||||
religious_character,17,Congregational Church
|
||||
religious_character,18,Free Church
|
||||
religious_character,19,Church of England/Free Church
|
||||
religious_character,20,Church of England/Christian
|
||||
religious_character,21,Sikh
|
||||
religious_character,22,Greek Orthodox
|
||||
religious_character,24,Buddhist
|
||||
religious_character,25,Hindu
|
||||
religious_character,26,Moravian
|
||||
religious_character,28,Inter- / non- denominational
|
||||
religious_character,29,Multi-faith
|
||||
religious_character,30,Church of England/Methodist/United Reform Church/Baptist
|
||||
religious_character,31,Anglican
|
||||
religious_character,32,Anglican/Christian
|
||||
religious_character,33,Anglican/Evangelical
|
||||
religious_character,34,Anglican/Church of England
|
||||
religious_character,35,Catholic
|
||||
religious_character,36,Charadi Jewish
|
||||
religious_character,37,Christian/Evangelical
|
||||
religious_character,38,Christian Science
|
||||
religious_character,39,Christian/Methodist
|
||||
religious_character,40,Christian/non-denominational
|
||||
religious_character,41,Church of England/Evangelical
|
||||
religious_character,42,Islam
|
||||
religious_character,43,Orthodox Jewish
|
||||
religious_character,44,Plymouth Brethren Christian Church
|
||||
religious_character,45,Protestant
|
||||
religious_character,46,Protestant/Evangelical
|
||||
religious_character,47,Reformed Baptist
|
||||
religious_character,48,Roman Catholic/Anglican
|
||||
religious_character,49,Sunni Deobandi
|
||||
admissions_policy,0,Not applicable
|
||||
admissions_policy,2,Selective
|
||||
admissions_policy,4,Non-selective
|
||||
|
@@ -1,33 +0,0 @@
|
||||
-- Warn when the live GIAS CSV carries a (code, name) pair we don't have in
|
||||
-- the dictionary seed — i.e. DfE added or renamed a value. Fix by rerunning
|
||||
-- pipeline/scripts/generate_gias_codes.py and committing the regenerated
|
||||
-- dictionaries + seed together.
|
||||
{{ config(severity='warn') }}
|
||||
|
||||
with raw_pairs as (
|
||||
{% for field_key, code_col, name_col in [
|
||||
('school_type', 'TypeOfEstablishment (code)', 'TypeOfEstablishment (name)'),
|
||||
('establishment_status', 'EstablishmentStatus (code)', 'EstablishmentStatus (name)'),
|
||||
('phase_of_education', 'PhaseOfEducation (code)', 'PhaseOfEducation (name)'),
|
||||
('official_sixth_form', 'OfficialSixthForm (code)', 'OfficialSixthForm (name)'),
|
||||
('religious_character', 'ReligiousCharacter (code)', 'ReligiousCharacter (name)'),
|
||||
('admissions_policy', 'AdmissionsPolicy (code)', 'AdmissionsPolicy (name)')
|
||||
] %}
|
||||
select distinct
|
||||
'{{ field_key }}' as field,
|
||||
cast(nullif(trim("{{ code_col }}"), '') as integer) as code,
|
||||
nullif(trim("{{ name_col }}"), '') as name
|
||||
from {{ source('raw', 'gias_establishments') }}
|
||||
where nullif(trim("{{ code_col }}"), '') is not null
|
||||
and nullif(trim("{{ name_col }}"), '') is not null
|
||||
{% if not loop.last %}union all{% endif %}
|
||||
{% endfor %}
|
||||
)
|
||||
|
||||
select r.*
|
||||
from raw_pairs r
|
||||
left join {{ ref('gias_code_names') }} s
|
||||
on s.field = r.field
|
||||
and s.code = r.code
|
||||
and s.name = r.name
|
||||
where s.field is null
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Comment-triggered PR fix-ups, powered by Claude Code.
|
||||
|
||||
Runs when a maintainer comments `@claude <instruction>` on a pull request.
|
||||
Checks out the PR head branch, hands the instruction to headless Claude Code
|
||||
(subscription OAuth auth — no API billing), commits and pushes whatever it
|
||||
changed (which re-runs the PR checks), and replies on the PR with a summary.
|
||||
|
||||
Uses only the Python standard library plus the `claude` CLI.
|
||||
|
||||
Required environment:
|
||||
CLAUDE_CODE_OAUTH_TOKEN token from `claude setup-token`
|
||||
GITEA_TOKEN run-scoped token (checkout auth handles the push)
|
||||
GITEA_SERVER_URL e.g. https://privaterepo.sitaru.org
|
||||
GITEA_REPOSITORY owner/repo
|
||||
PR_NUMBER pull request index
|
||||
COMMENT_BODY the triggering comment text
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
TRIGGER = "@claude"
|
||||
CLAUDE_TIMEOUT_S = 2400 # 40 min ceiling for one fix-up session
|
||||
|
||||
|
||||
def api(path: str, payload: dict | None = None) -> dict:
|
||||
server = os.environ["GITEA_SERVER_URL"].rstrip("/")
|
||||
repo = os.environ["GITEA_REPOSITORY"]
|
||||
req = urllib.request.Request(
|
||||
f"{server}/api/v1/repos/{repo}{path}",
|
||||
data=json.dumps(payload).encode() if payload is not None else None,
|
||||
headers={
|
||||
"Authorization": f"token {os.environ['GITEA_TOKEN']}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST" if payload is not None else "GET",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def run(*cmd: str, **kwargs) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(cmd, check=True, capture_output=True, text=True, **kwargs)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pr_number = os.environ["PR_NUMBER"]
|
||||
instruction = os.environ["COMMENT_BODY"].strip()
|
||||
if instruction.lower().startswith(TRIGGER):
|
||||
instruction = instruction[len(TRIGGER):].strip()
|
||||
if not instruction:
|
||||
print("Empty instruction after trigger word; nothing to do")
|
||||
return 0
|
||||
|
||||
pr = api(f"/pulls/{pr_number}")
|
||||
head_ref = pr["head"]["ref"]
|
||||
base_ref = pr["base"]["ref"]
|
||||
|
||||
run("git", "fetch", "origin", head_ref, base_ref)
|
||||
run("git", "checkout", head_ref)
|
||||
|
||||
prompt = f"""You are working on pull request #{pr_number} ("{pr['title']}")
|
||||
in the SchoolCompare repository. The PR branch is checked out; its base is
|
||||
{base_ref}. A maintainer left this instruction on the PR:
|
||||
|
||||
{instruction}
|
||||
|
||||
Implement exactly what was asked, following the conventions in CLAUDE.md.
|
||||
Run any relevant tests or typechecks you can. Do NOT commit or push — the
|
||||
harness handles that. When done, summarise in a few sentences what you
|
||||
changed and how you verified it."""
|
||||
|
||||
proc = subprocess.run(
|
||||
["claude", "-p", prompt, "--dangerously-skip-permissions", "--output-format", "json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=CLAUDE_TIMEOUT_S,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"claude CLI failed:\n{proc.stderr[-2000:]}")
|
||||
summary = json.loads(proc.stdout)["result"].strip()
|
||||
|
||||
changed = run("git", "status", "--porcelain").stdout.strip()
|
||||
if changed:
|
||||
run("git", "config", "user.name", "Claude (CI)")
|
||||
run("git", "config", "user.email", "noreply@anthropic.com")
|
||||
run("git", "add", "-A")
|
||||
title = instruction.splitlines()[0][:60]
|
||||
run("git", "commit", "-m", f"ai: {title}\n\nRequested via PR comment; applied by Claude Code in CI.")
|
||||
run("git", "push", "origin", head_ref)
|
||||
sha = run("git", "rev-parse", "--short", "HEAD").stdout.strip()
|
||||
reply = f"## 🤖 Claude fix-up applied (`{sha}`)\n\n{summary}\n\n_PR checks re-run automatically on the new commit._"
|
||||
else:
|
||||
reply = f"## 🤖 Claude fix-up — no changes made\n\n{summary}"
|
||||
|
||||
api(f"/issues/{pr_number}/comments", {"body": reply})
|
||||
print(reply)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,5 +0,0 @@
|
||||
-- Retire the Ofsted Parent View feature (schema v6).
|
||||
-- The marts schema is dbt-owned; deleting the dbt model stops the table being
|
||||
-- rebuilt but does not drop the existing relation, so apply this directly
|
||||
-- against the staging and production marts databases.
|
||||
DROP TABLE IF EXISTS marts.fact_parent_view CASCADE;
|
||||
Reference in New Issue
Block a user