.gitea/workflows/deploy.yml

@@ -1,12 +1,9 @@
-name: Build and Push Docker Images
+name: Deploy (staging -> E2E gate -> production)
 
 on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
 
 env:
   REGISTRY: privaterepo.sitaru.org
@@ -45,17 +42,15 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}
           tags: |
-            type=ref,event=branch
+            type=sha
-            type=ref,event=pr
+            type=raw,value=staging
-            type=sha,prefix=backend-
-            type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
 
       - name: Build and push Backend Docker image
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
-          push: ${{ gitea.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta-backend.outputs.tags }}
           labels: ${{ steps.meta-backend.outputs.labels }}
           cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:buildcache
@@ -91,24 +86,20 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.FRONTEND_IMAGE_NAME }}
           tags: |
-            type=ref,event=branch
+            type=sha
-            type=ref,event=pr
+            type=raw,value=staging
-            type=sha,prefix=frontend-
-            type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
 
       - name: Build and push Frontend Docker image
         uses: docker/build-push-action@v5
         with:
           context: ./nextjs-app
           file: ./nextjs-app/Dockerfile
-          push: ${{ gitea.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta-frontend.outputs.tags }}
           labels: ${{ steps.meta-frontend.outputs.labels }}
           build-args: |
             FASTAPI_URL=http://backend:80/api
           # Cache disabled due to registry size limits
-          # cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.FRONTEND_IMAGE_NAME }}:buildcache
-          # cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.FRONTEND_IMAGE_NAME }}:buildcache,mode=max
 
   build-pipeline:
     name: Build Pipeline (Meltano + dbt + Airflow)
@@ -140,28 +131,110 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.PIPELINE_IMAGE_NAME }}
           tags: |
-            type=ref,event=branch
+            type=sha
-            type=ref,event=pr
+            type=raw,value=staging
-            type=sha,prefix=pipeline-
-            type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
 
       - name: Build and push Pipeline Docker image
         uses: docker/build-push-action@v5
         with:
           context: ./pipeline
           file: ./pipeline/Dockerfile
-          push: ${{ gitea.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta-pipeline.outputs.tags }}
           labels: ${{ steps.meta-pipeline.outputs.labels }}
           cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PIPELINE_IMAGE_NAME }}:buildcache
           cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PIPELINE_IMAGE_NAME }}:buildcache,mode=max
 
-  trigger-deployment:
+  deploy-staging:
-    name: Trigger Portainer Update
+    name: Deploy to Staging
     runs-on: ubuntu-latest
     needs: [build-backend, build-frontend, build-pipeline]
-    if: gitea.event_name != 'pull_request'
     steps:
-      - name: Trigger Portainer stack update
+      - name: Trigger staging stack update
+        run: curl -fsSk -X POST "${{ secrets.PORTAINER_STAGING_WEBHOOK }}"
+
+      - name: Wait for staging to become healthy
         run: |
-          curl -X POST -k "https://10.0.1.224:9443/api/stacks/webhooks/863fc57c-bf24-4c63-9001-bdf9912fba73"
+          echo "Polling ${STAGING_BASE_URL} for up to 5 minutes..."
+          for i in $(seq 1 60); do
+            if curl -fsS -o /dev/null --max-time 10 "${STAGING_BASE_URL}/"; then
+              echo "Staging is up (attempt $i)"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "Staging did not become healthy in time" >&2
+          exit 1
+        env:
+          STAGING_BASE_URL: ${{ secrets.STAGING_BASE_URL }}
+
+  e2e-staging:
+    name: E2E Journeys against Staging
+    runs-on: ubuntu-latest
+    needs: [deploy-staging]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install Playwright
+        working-directory: e2e
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Run E2E journeys
+        working-directory: e2e
+        run: npx playwright test
+        env:
+          BASE_URL: ${{ secrets.STAGING_BASE_URL }}
+
+  promote-prod:
+    name: Promote to Production
+    runs-on: ubuntu-latest
+    needs: [e2e-staging]
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ gitea.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Retag verified images as prod
+        run: |
+          SHORT_SHA="sha-$(echo "${{ gitea.sha }}" | cut -c1-7)"
+          for IMAGE in \
+            "${REGISTRY}/${BACKEND_IMAGE_NAME}" \
+            "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \
+            "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do
+            # Keep a rollback pointer before moving :prod
+            docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true
+            docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}"
+            echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod"
+          done
+
+      - name: Trigger production stack update
+        run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}"
+
+      - name: Wait for production to become healthy
+        run: |
+          echo "Polling ${PROD_BASE_URL} for up to 5 minutes..."
+          for i in $(seq 1 60); do
+            if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then
+              echo "Production is up (attempt $i)"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "Production did not become healthy in time" >&2
+          exit 1
+        env:
+          PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }}

.gitea/workflows/pr-checks.yml

@@ -0,0 +1,181 @@
+name: PR Checks
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  REGISTRY: privaterepo.sitaru.org
+  BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend
+  FRONTEND_IMAGE_NAME: ${{ gitea.repository }}-frontend
+  PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline
+
+jobs:
+  frontend-checks:
+    name: Frontend Typecheck + Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: nextjs-app/package-lock.json
+
+      - name: Install dependencies
+        working-directory: nextjs-app
+        run: npm ci
+
+      - name: Typecheck
+        working-directory: nextjs-app
+        run: npm run typecheck
+
+      - name: Unit tests
+        working-directory: nextjs-app
+        run: npm test
+
+  backend-checks:
+    name: Backend Smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Import smoke test
+        run: python -c "from backend.app import app; print('backend imports OK')"
+
+  build-backend:
+    name: Build Backend (no push)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["10.0.1.224:6000"]
+            [registry."10.0.1.224:6000"]
+              http = true
+              insecure = true
+
+      - name: Log in to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ gitea.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Build Backend Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          push: false
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:buildcache
+
+  build-frontend:
+    name: Build Frontend (no push)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["10.0.1.224:6000"]
+            [registry."10.0.1.224:6000"]
+              http = true
+              insecure = true
+
+      - name: Log in to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ gitea.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Build Frontend Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./nextjs-app
+          file: ./nextjs-app/Dockerfile
+          push: false
+          build-args: |
+            FASTAPI_URL=http://backend:80/api
+
+  build-pipeline:
+    name: Build Pipeline (no push)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["10.0.1.224:6000"]
+            [registry."10.0.1.224:6000"]
+              http = true
+              insecure = true
+
+      - name: Log in to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ gitea.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Build Pipeline Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./pipeline
+          file: ./pipeline/Dockerfile
+          push: false
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PIPELINE_IMAGE_NAME }}:buildcache
+
+  ai-review:
+    name: AI Code Review (Claude)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install anthropic requests
+
+      - name: Review PR diff with Claude
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          GITEA_SERVER_URL: ${{ gitea.server_url }}
+          GITEA_REPOSITORY: ${{ gitea.repository }}
+          PR_NUMBER: ${{ gitea.event.pull_request.number }}
+          BASE_REF: ${{ gitea.event.pull_request.base.ref }}
+        run: python scripts/ci/ai_review.py

claude.md

@@ -105,8 +105,22 @@ This starts:
 - `GET /api/metrics` - Metric definitions (single source of truth)
 - `GET /api/data-info` - Database stats
 
+## SDLC
+
+Full details in `docs/DEPLOY.md`. The short version:
+
+- **Never push to `main` directly.** Work on a feature branch and open a PR;
+  branch protection requires the PR checks (typecheck, tests, builds, AI review)
+  to pass before merge.
+- Merging to `main` deploys automatically: images are built once, deployed to
+  the **staging** Portainer stack, verified by the Playwright journeys in
+  `e2e/`, and only then retagged `:prod` and rolled out to production.
+- If you change user-facing behaviour, update or extend the `e2e/` journey
+  tests in the same PR — they are the promotion gate.
+
 ## Recent Changes
 
+- Added staging environment + automated staging→prod pipeline (Gitea Actions)
 - Migrated from CSV file storage to PostgreSQL database
 - Added location-based search using postcode geocoding
 - Added local authority filter to rankings

docker-compose.portainer.staging.yml

@@ -0,0 +1,214 @@
+# Portainer Stack Definition for School Compare — STAGING
+#
+# Deploy this as a *separate* Portainer stack (e.g. "schoolcompare-staging")
+# alongside the production stack. Differences from production:
+#   - images pinned to :staging (pushed by every merge to main, before the E2E gate)
+#   - sc_staging_* container names
+#   - own macvlan IPs (STAGING_DB_IP / STAGING_FRONTEND_IP env vars)
+#   - Airflow UI published on 8081 (prod uses 8080)
+#   - volumes are isolated automatically: Portainer prefixes volume names with
+#     the stack name, so this stack gets its own postgres/typesense/airflow data
+#
+# Portainer environment variables (set in Portainer UI -> Stack -> Environment):
+#   DB_USERNAME            — PostgreSQL username
+#   DB_PASSWORD            — PostgreSQL password
+#   DB_DATABASE_NAME       — PostgreSQL database name
+#   ADMIN_API_KEY          — Backend admin API key
+#   TYPESENSE_API_KEY      — Typesense admin API key
+#   TYPESENSE_SEARCH_KEY   — Typesense search-only key (exposed to frontend)
+#   AIRFLOW_ADMIN_USER     — Airflow admin username (password auto-generated, see api-server logs)
+#   STAGING_DB_IP          — macvlan IP for staging Postgres (default 10.0.1.190)
+#   STAGING_FRONTEND_IP    — macvlan IP for staging frontend (default 10.0.1.151)
+
+services:
+
+  # ── PostgreSQL ────────────────────────────────────────────────────────
+  sc_database:
+    container_name: sc_staging_postgres
+    image: postgis/postgis:18-3.6-alpine
+    environment:
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_USER: ${DB_USERNAME}
+      POSTGRES_DB: ${DB_DATABASE_NAME}
+    volumes:
+      - postgres_data:/var/lib/postgresql
+    shm_size: 128mb
+    networks:
+      backend: {}
+      macvlan:
+        ipv4_address: ${STAGING_DB_IP:-10.0.1.190}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    restart: unless-stopped
+
+  # ── FastAPI Backend ───────────────────────────────────────────────────
+  backend:
+    image: privaterepo.sitaru.org/tudor/school_compare-backend:staging
+    container_name: sc_staging_backend
+    environment:
+      DATABASE_URL: postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
+      PYTHONUNBUFFERED: 1
+      ADMIN_API_KEY: ${ADMIN_API_KEY:-changeme}
+      TYPESENSE_URL: http://typesense:8108
+      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
+    depends_on:
+      sc_database:
+        condition: service_healthy
+    networks:
+      - backend
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/api/data-info"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+  # ── Next.js Frontend ──────────────────────────────────────────────────
+  frontend:
+    image: privaterepo.sitaru.org/tudor/school_compare-frontend:staging
+    container_name: sc_staging_nextjs
+    environment:
+      - NODE_ENV=production
+      - NEXT_PUBLIC_API_URL=http://localhost:8000/api
+      - FASTAPI_URL=http://backend:80/api
+      - TYPESENSE_URL=http://typesense:8108
+      - TYPESENSE_API_KEY=${TYPESENSE_SEARCH_KEY:-changeme}
+    depends_on:
+      backend:
+        condition: service_healthy
+    networks:
+      backend: {}
+      macvlan:
+        ipv4_address: ${STAGING_FRONTEND_IP:-10.0.1.151}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+  # ── Typesense Search Engine ───────────────────────────────────────────
+  typesense:
+    image: typesense/typesense:30.1
+    container_name: sc_staging_typesense
+    environment:
+      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
+      TYPESENSE_DATA_DIR: /data
+    volumes:
+      - typesense_data:/data
+    networks:
+      - backend
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "cat < /dev/tcp/localhost/8108"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+
+  # ── Airflow API Server + UI (staging: http://<host>:8081) ─────────────
+  airflow-api-server:
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:staging
+    container_name: sc_staging_airflow_api
+    command: airflow api-server --port 8080
+    ports:
+      - "8081:8080"
+    environment:
+      AIRFLOW__CORE__EXECUTOR: LocalExecutor
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
+      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
+      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-api-server:8080/execution/
+      AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-staging-airflow-jwt-secret-key-long-enough-for-sha512"
+      AIRFLOW__API_AUTH__JWT_ISSUER: airflow
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
+      AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
+      PG_HOST: sc_database
+      PG_PORT: "5432"
+      PG_USER: ${DB_USERNAME}
+      PG_PASSWORD: ${DB_PASSWORD}
+      PG_DATABASE: ${DB_DATABASE_NAME}
+      TYPESENSE_URL: http://typesense:8108
+      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
+    volumes:
+      - airflow_logs:/opt/airflow/logs
+    depends_on:
+      sc_database:
+        condition: service_healthy
+    networks:
+      - backend
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/api/v2/monitor/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  # ── Airflow Scheduler ──────────────────────────────────────────────
+  airflow-scheduler:
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:staging
+    container_name: sc_staging_airflow_scheduler
+    command: airflow scheduler
+    environment:
+      AIRFLOW__CORE__EXECUTOR: LocalExecutor
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
+      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
+      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-api-server:8080/execution/
+      AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-staging-airflow-jwt-secret-key-long-enough-for-sha512"
+      AIRFLOW__API_AUTH__JWT_ISSUER: airflow
+      AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
+      PG_HOST: sc_database
+      PG_PORT: "5432"
+      PG_USER: ${DB_USERNAME}
+      PG_PASSWORD: ${DB_PASSWORD}
+      PG_DATABASE: ${DB_DATABASE_NAME}
+      TYPESENSE_URL: http://typesense:8108
+      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme}
+    volumes:
+      - airflow_logs:/opt/airflow/logs
+    depends_on:
+      sc_database:
+        condition: service_healthy
+    networks:
+      - backend
+    restart: unless-stopped
+
+  # ── Airflow DB Init (one-shot) ───────────────────────────────────────
+  airflow-init:
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:staging
+    container_name: sc_staging_airflow_init
+    command: bash -c "airflow db migrate && airflow dags reserialize"
+    environment:
+      AIRFLOW__CORE__EXECUTOR: LocalExecutor
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
+      AIRFLOW__CORE__DAGS_FOLDER: /opt/pipeline/dags
+      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-api-server:8080/execution/
+      AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-staging-airflow-jwt-secret-key-long-enough-for-sha512"
+      AIRFLOW__API_AUTH__JWT_ISSUER: airflow
+    depends_on:
+      sc_database:
+        condition: service_healthy
+    networks:
+      - backend
+    restart: "no"
+
+networks:
+  backend:
+    driver: bridge
+  macvlan:
+    external:
+      name: macvlan
+
+volumes:
+  postgres_data:
+  typesense_data:
+  airflow_logs:

docker-compose.portainer.yml

@@ -36,7 +36,7 @@ services:
 
   # ── FastAPI Backend ───────────────────────────────────────────────────
   backend:
-    image: privaterepo.sitaru.org/tudor/school_compare-backend:latest
+    image: privaterepo.sitaru.org/tudor/school_compare-backend:prod
     container_name: schoolcompare_backend
     environment:
       DATABASE_URL: postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME}
@@ -59,7 +59,7 @@ services:
 
   # ── Next.js Frontend ──────────────────────────────────────────────────
   frontend:
-    image: privaterepo.sitaru.org/tudor/school_compare-frontend:latest
+    image: privaterepo.sitaru.org/tudor/school_compare-frontend:prod
     container_name: schoolcompare_nextjs
     environment:
       - NODE_ENV=production
@@ -103,7 +103,7 @@ services:
 
   # ── Airflow API Server + UI ───────────────────────────────────────────
   airflow-api-server:
-    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
     container_name: schoolcompare_airflow_api
     command: airflow api-server --port 8080
     ports:
@@ -142,7 +142,7 @@ services:
 
   # ── Airflow Scheduler ──────────────────────────────────────────────
   airflow-scheduler:
-    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
     container_name: schoolcompare_airflow_scheduler
     command: airflow scheduler
     environment:
@@ -172,7 +172,7 @@ services:
 
   # ── Airflow DB Init (one-shot) ───────────────────────────────────────
   airflow-init:
-    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
+    image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
     container_name: schoolcompare_airflow_init
     command: bash -c "airflow db migrate && airflow dags delete school_data_daily -y 2>/dev/null; airflow dags delete school_data_monthly_ofsted -y 2>/dev/null; airflow dags delete school_data_annual_ees -y 2>/dev/null; airflow dags reserialize"
     environment:

docs/DEPLOY.md

@@ -0,0 +1,130 @@
+# SDLC & Deployment Pipeline
+
+SchoolCompare uses a fully automated staging → production pipeline on Gitea
+Actions. AI writes the code on feature branches; the pipeline verifies every
+change on a staging environment before promoting the exact same images to
+production. Human input is directional only: feature requests, PR review if
+desired, and intervention when a gate fails.
+
+## The flow
+
+```
+feature branch (AI-authored)
+   │  PR to main
+   ▼
+PR checks (.gitea/workflows/pr-checks.yml)
+   typecheck + unit tests + backend smoke + image builds (no push)
+   + Claude code review posted as a PR comment (severe findings fail the check)
+   │  merge (branch protection requires green checks)
+   ▼
+Deploy pipeline (.gitea/workflows/deploy.yml)
+   1. build & push images        → tags sha-<sha>, staging
+   2. staging Portainer webhook  → wait for staging health
+   3. Playwright E2E journeys against staging
+   4. retag sha-<sha> → :prod    (same bytes — build once, promote the image)
+      previous :prod saved as :prod-previous
+   5. prod Portainer webhook     → wait for prod health
+```
+
+Key principle: **build once, promote the exact image**. Production pins `:prod`,
+which only moves after the E2E gate passes on staging. Nothing tags `:latest`
+anymore.
+
+## Branch & PR workflow
+
+- `main` is protected: no direct pushes, PRs require green status checks.
+- All work (human or AI) happens on feature branches → PR to `main`.
+- Merging to `main` **is** the release action. If staging or the E2E gate
+  fails, production is untouched.
+
+## Environments
+
+| | Production | Staging |
+|---|---|---|
+| Portainer stack file | `docker-compose.portainer.yml` | `docker-compose.portainer.staging.yml` |
+| Image tag | `:prod` | `:staging` |
+| Container prefix | `sc_` / `schoolcompare_` | `sc_staging_` |
+| Frontend macvlan IP | 10.0.1.150 | `STAGING_FRONTEND_IP` (default 10.0.1.151) |
+| Postgres macvlan IP | 10.0.1.189 | `STAGING_DB_IP` (default 10.0.1.190) |
+| Airflow UI port | 8080 | 8081 |
+| Volumes | stack-prefixed | stack-prefixed (fully isolated) |
+
+Staging gets `:staging` images on every merge to main — even ones that later
+fail the E2E gate. That's the point: staging absorbs the risk.
+
+## Gitea repository secrets
+
+| Secret | Purpose |
+|---|---|
+| `REGISTRY_TOKEN` | push images to privaterepo.sitaru.org (already set) |
+| `ANTHROPIC_API_KEY` | Claude PR review (`scripts/ci/ai_review.py`) |
+| `GITEA_TOKEN` | post PR review comments (needs issue-comment scope) |
+| `PORTAINER_STAGING_WEBHOOK` | staging stack redeploy webhook URL |
+| `PORTAINER_PROD_WEBHOOK` | production stack redeploy webhook URL |
+| `STAGING_BASE_URL` | e.g. `http://10.0.1.151:3000` — health poll + E2E target |
+| `PROD_BASE_URL` | e.g. `http://10.0.1.150:3000` — post-promotion health poll |
+
+## One-time setup checklist
+
+1. **Create the staging stack** in Portainer from
+   `docker-compose.portainer.staging.yml` (stack name e.g.
+   `schoolcompare-staging`). Set the same environment variables as prod plus
+   `STAGING_DB_IP` / `STAGING_FRONTEND_IP` if the defaults clash.
+2. **Enable webhooks** on both stacks (Portainer → Stack → Webhook) and store
+   the URLs as `PORTAINER_STAGING_WEBHOOK` / `PORTAINER_PROD_WEBHOOK`. Remove
+   the old hardcoded webhook usage (now gone from the workflows).
+3. **Add the remaining secrets** listed above in Gitea → repo → Settings →
+   Actions → Secrets.
+4. **Protect `main`** in Gitea → Settings → Branches: require PRs, require the
+   pr-checks status checks (frontend, backend, builds, ai-review) to pass.
+5. **Bootstrap staging data via Airflow** (no prod dump — staging populates
+   itself from source, exercising the pipeline image end-to-end):
+   - Open the staging Airflow UI (`http://<host>:8081`) and trigger, in order:
+     `school_data_daily`, `school_data_monthly_ofsted`,
+     `school_data_monthly_parent_view`, then the manual-schedule
+     `school_data_annual_ees` and `school_data_annual_idaci`.
+   - First runs download from government sources (GIAS, Ofsted, EES, IDACI),
+     run dbt, and sync Typesense — expect the initial backfill to take a while.
+   - The scheduled DAGs then keep staging fresh exactly like prod.
+6. **Switch the prod stack to `:prod` tags** — the repo's
+   `docker-compose.portainer.yml` is already updated; redeploy the prod stack
+   from it. Until the first pipeline run promotes an image, tag the current
+   images manually: `docker buildx imagetools create -t <image>:prod <image>:latest`
+   for each of the three images.
+
+## Rollback
+
+Every promotion first re-points `:prod-previous` at the outgoing `:prod`.
+To roll back:
+
+```bash
+for img in backend frontend pipeline; do
+  docker buildx imagetools create \
+    -t privaterepo.sitaru.org/tudor/school_compare-$img:prod \
+    privaterepo.sitaru.org/tudor/school_compare-$img:prod-previous
+done
+curl -fsSk -X POST "$PORTAINER_PROD_WEBHOOK"
+```
+
+Or promote any older build directly: `imagetools create -t <image>:prod <image>:sha-<shortsha>`.
+
+## E2E suite
+
+Lives in `e2e/` (own package — CI installs it without the app's node_modules).
+Journeys: home + name search, postcode search, school detail, two-school
+comparison, rankings table. Run locally against any environment:
+
+```bash
+cd e2e && npm ci
+BASE_URL=http://10.0.1.151:3000 npx playwright test
+```
+
+Tests assert data invariants (results exist, charts render), not exact
+numbers, so scheduled data refreshes don't break the gate.
+
+## AI code review
+
+`scripts/ci/ai_review.py` sends the PR diff to Claude (`claude-opus-4-8`),
+posts the structured findings as a PR comment, and fails the check only when a
+finding is rated **severe** (would break prod, leak data, or corrupt data).
+Minor findings are informational and never block a merge.

e2e/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+test-results/
+playwright-report/

e2e/package-lock.json

@@ -0,0 +1,78 @@
+{
+  "name": "schoolcompare-e2e",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "schoolcompare-e2e",
+      "version": "1.0.0",
+      "devDependencies": {
+        "@playwright/test": "^1.49.0"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz",
+      "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.61.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",
+      "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.61.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",
+      "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

e2e/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "schoolcompare-e2e",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Journey tests run against staging as the production promotion gate",
+  "scripts": {
+    "test": "playwright test"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.49.0"
+  }
+}

e2e/playwright.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  timeout: 60_000,
+  retries: 1,
+  workers: 2,
+  reporter: process.env.CI ? 'list' : 'html',
+  use: {
+    baseURL: process.env.BASE_URL || 'http://localhost:3000',
+    trace: 'retain-on-failure',
+  },
+});

e2e/tests/journeys.spec.ts

@@ -0,0 +1,72 @@
+import { test, expect, Page } from '@playwright/test';
+
+/**
+ * Journey tests for SchoolCompare, run against the staging environment as the
+ * gate before promotion to production. They assert stable data invariants
+ * (results exist, key UI renders) rather than exact numbers, so routine data
+ * refreshes don't break the pipeline.
+ */
+
+async function searchByName(page: Page, query: string) {
+  await page.goto('/');
+  const searchInput = page.getByPlaceholder('School name or postcode').first();
+  await searchInput.fill(query);
+  await searchInput.press('Enter');
+  await page.waitForURL(/search=|postcode=/);
+}
+
+function schoolLinks(page: Page) {
+  return page.locator('a[href^="/school/"]');
+}
+
+test('home page loads with hero search', async ({ page }) => {
+  await page.goto('/');
+  await expect(page.locator('h1').first()).toBeVisible();
+  await expect(page.getByPlaceholder('School name or postcode').first()).toBeVisible();
+});
+
+test('searching by name returns school results', async ({ page }) => {
+  await searchByName(page, 'primary');
+  await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 });
+  expect(await schoolLinks(page).count()).toBeGreaterThan(1);
+});
+
+test('searching by postcode returns nearby schools', async ({ page }) => {
+  await searchByName(page, 'B1 1BB');
+  await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 });
+});
+
+test('school detail page renders name and performance data', async ({ page }) => {
+  await searchByName(page, 'primary');
+  const firstSchool = schoolLinks(page).first();
+  await expect(firstSchool).toBeVisible({ timeout: 15_000 });
+  await firstSchool.click();
+  await page.waitForURL(/\/school\//);
+  await expect(page.locator('h1').first()).toBeVisible();
+  // The detail page renders at least one chart canvas (performance history)
+  await expect(page.locator('canvas').first()).toBeVisible({ timeout: 15_000 });
+});
+
+test('comparing two schools shows both side by side', async ({ page }) => {
+  // Collect two school URNs from search results, then load the share URL
+  await searchByName(page, 'primary');
+  await expect(schoolLinks(page).first()).toBeVisible({ timeout: 15_000 });
+  const hrefs = await schoolLinks(page).evaluateAll((links) =>
+    links.map((l) => (l as HTMLAnchorElement).getAttribute('href') || '')
+  );
+  const urns = [...new Set(hrefs.map((h) => h.match(/\/school\/(\d+)/)?.[1]).filter(Boolean))];
+  expect(urns.length).toBeGreaterThanOrEqual(2);
+
+  await page.goto(`/compare?urns=${urns[0]},${urns[1]}`);
+  // Both schools' detail links should render in the comparison view
+  await expect(page.locator(`a[href*="${urns[0]}"]`).first()).toBeVisible({ timeout: 15_000 });
+  await expect(page.locator(`a[href*="${urns[1]}"]`).first()).toBeVisible();
+});
+
+test('rankings page loads a populated table', async ({ page }) => {
+  await page.goto('/rankings');
+  await expect(page.getByRole('heading', { name: /rankings/i }).first()).toBeVisible();
+  const rows = page.locator('table tbody tr');
+  await expect(rows.first()).toBeVisible({ timeout: 15_000 });
+  expect(await rows.count()).toBeGreaterThan(5);
+});

nextjs-app/__tests__/components/SchoolCard.test.tsx

@@ -34,24 +34,24 @@ describe('SchoolCard', () => {
     render(<SchoolCard school={mockSchool} />);
 
     const link = screen.getByRole('link', { name: /test primary school/i });
-    expect(link).toHaveAttribute('href', '/school/100001');
+    expect(link).toHaveAttribute('href', '/school/100001-test-primary-school');
   });
 
-  it('calls onAddToCompare when Add to Compare button is clicked', () => {
+  it('calls onAddToCompare when the Compare button is clicked', () => {
     const mockAddToCompare = jest.fn();
     render(<SchoolCard school={mockSchool} onAddToCompare={mockAddToCompare} />);
 
-    const addButton = screen.getByText('Add to Compare');
+    const addButton = screen.getByText('+ Compare');
     fireEvent.click(addButton);
 
     expect(mockAddToCompare).toHaveBeenCalledWith(mockSchool);
     expect(mockAddToCompare).toHaveBeenCalledTimes(1);
   });
 
-  it('does not render Add to Compare button when handler not provided', () => {
+  it('does not render the Compare button when handler not provided', () => {
     render(<SchoolCard school={mockSchool} />);
 
-    expect(screen.queryByText('Add to Compare')).not.toBeInTheDocument();
+    expect(screen.queryByText('+ Compare')).not.toBeInTheDocument();
   });
 
   it('displays trend indicator for positive change', () => {

nextjs-app/__tests__/lib/utils.test.ts

@@ -19,7 +19,7 @@ describe('formatPercentage', () => {
   });
 
   it('handles null values', () => {
-    expect(formatPercentage(null)).toBe('-');
+    expect(formatPercentage(null)).toBe('N/A');
   });
 });
 
@@ -31,7 +31,7 @@ describe('formatProgress', () => {
   });
 
   it('handles null values', () => {
-    expect(formatProgress(null)).toBe('-');
+    expect(formatProgress(null)).toBe('N/A');
   });
 });
 
@@ -44,16 +44,16 @@ describe('calculateTrend', () => {
     expect(calculateTrend(70, 75)).toBe('down');
   });
 
-  it('calculates same trend', () => {
+  it('calculates stable trend', () => {
-    expect(calculateTrend(75, 75)).toBe('same');
+    expect(calculateTrend(75, 75)).toBe('stable');
   });
 
   it('handles null previous value', () => {
-    expect(calculateTrend(75, null)).toBe('same');
+    expect(calculateTrend(75, null)).toBe('stable');
   });
 
   it('handles null current value', () => {
-    expect(calculateTrend(null, 75)).toBe('same');
+    expect(calculateTrend(null, 75)).toBe('stable');
   });
 });
 
@@ -72,7 +72,13 @@ describe('isValidPostcode', () => {
 });
 
 describe('debounce', () => {
+  beforeEach(() => {
     jest.useFakeTimers();
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
 
   it('delays function execution', () => {
     const mockFn = jest.fn();
@@ -100,8 +106,6 @@ describe('debounce', () => {
     expect(mockFn).toHaveBeenCalledWith('third');
     expect(mockFn).toHaveBeenCalledTimes(1);
   });
-
-  jest.useRealTimers();
 });
 
 describe('buildOfstedListBadge', () => {

nextjs-app/package.json

@@ -7,7 +7,7 @@
     "dev": "next dev",
     "build": "next build",
     "start": "next start",
-    "lint": "next lint",
+    "typecheck": "tsc --noEmit",
     "test": "jest",
     "test:watch": "jest --watch",
     "test:coverage": "jest --coverage"

scripts/ci/ai_review.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""AI code review for Gitea pull requests.
+
+Reads the PR diff (base branch vs HEAD), asks Claude to review it, posts the
+findings as a PR comment via the Gitea API, and exits non-zero only when the
+review contains at least one severe finding — so the job can gate merges
+without blocking on nitpicks.
+
+Required environment:
+    ANTHROPIC_API_KEY   Anthropic API key
+    GITEA_TOKEN         Gitea token with permission to comment on PRs
+    GITEA_SERVER_URL    e.g. https://privaterepo.sitaru.org
+    GITEA_REPOSITORY    owner/repo
+    PR_NUMBER           pull request index
+    BASE_REF            base branch name (e.g. main)
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+import requests
+from anthropic import Anthropic
+
+MAX_DIFF_CHARS = 150_000
+
+REVIEW_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "summary": {
+            "type": "string",
+            "description": "Two or three sentences on what the change does and its overall health.",
+        },
+        "findings": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "severity": {"type": "string", "enum": ["severe", "minor"]},
+                    "file": {"type": "string"},
+                    "issue": {"type": "string"},
+                },
+                "required": ["severity", "file", "issue"],
+                "additionalProperties": False,
+            },
+        },
+    },
+    "required": ["summary", "findings"],
+    "additionalProperties": False,
+}
+
+SYSTEM_PROMPT = """You are reviewing a pull request for SchoolCompare, a UK school
+comparison site (FastAPI backend, Next.js frontend, Airflow/dbt data pipeline,
+deployed via Gitea Actions to a staging-then-production Docker setup).
+
+Report correctness bugs, security issues, data-loss risks, and broken deploy/CI
+configuration. Mark a finding "severe" only if it would break production, leak
+data, or corrupt data — severe findings block the merge. Everything else
+(style, performance suggestions, minor cleanups) is "minor". Do not invent
+findings: an empty findings list is a perfectly good review of a clean diff."""
+
+
+def get_diff(base_ref: str) -> str:
+    subprocess.run(
+        ["git", "fetch", "origin", base_ref],
+        check=True,
+        capture_output=True,
+    )
+    diff = subprocess.run(
+        ["git", "diff", f"origin/{base_ref}...HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    if len(diff) > MAX_DIFF_CHARS:
+        diff = diff[:MAX_DIFF_CHARS] + "\n\n[diff truncated for review]"
+    return diff
+
+
+def review(diff: str) -> dict:
+    client = Anthropic()
+    with client.messages.stream(
+        model="claude-opus-4-8",
+        max_tokens=16000,
+        thinking={"type": "adaptive"},
+        system=SYSTEM_PROMPT,
+        output_config={"format": {"type": "json_schema", "schema": REVIEW_SCHEMA}},
+        messages=[
+            {
+                "role": "user",
+                "content": f"Review this pull request diff:\n\n```diff\n{diff}\n```",
+            }
+        ],
+    ) as stream:
+        message = stream.get_final_message()
+    if message.stop_reason == "refusal":
+        raise RuntimeError("Claude declined to review this diff")
+    text = next(b.text for b in message.content if b.type == "text")
+    return json.loads(text)
+
+
+def format_comment(result: dict) -> str:
+    lines = ["## 🤖 AI Code Review (Claude)", "", result["summary"], ""]
+    severe = [f for f in result["findings"] if f["severity"] == "severe"]
+    minor = [f for f in result["findings"] if f["severity"] == "minor"]
+    if severe:
+        lines.append("### 🔴 Severe (blocks merge)")
+        lines += [f"- **{f['file']}**: {f['issue']}" for f in severe]
+        lines.append("")
+    if minor:
+        lines.append("### 🟡 Minor")
+        lines += [f"- **{f['file']}**: {f['issue']}" for f in minor]
+        lines.append("")
+    if not result["findings"]:
+        lines.append("✅ No issues found.")
+    return "\n".join(lines)
+
+
+def post_comment(body: str) -> None:
+    server = os.environ["GITEA_SERVER_URL"].rstrip("/")
+    repo = os.environ["GITEA_REPOSITORY"]
+    pr = os.environ["PR_NUMBER"]
+    resp = requests.post(
+        f"{server}/api/v1/repos/{repo}/issues/{pr}/comments",
+        headers={"Authorization": f"token {os.environ['GITEA_TOKEN']}"},
+        json={"body": body},
+        timeout=30,
+    )
+    resp.raise_for_status()
+
+
+def main() -> int:
+    diff = get_diff(os.environ["BASE_REF"])
+    if not diff.strip():
+        print("Empty diff, nothing to review")
+        return 0
+    result = review(diff)
+    comment = format_comment(result)
+    print(comment)
+    post_comment(comment)
+    severe = [f for f in result["findings"] if f["severity"] == "severe"]
+    if severe:
+        print(f"\n{len(severe)} severe finding(s) — failing the check", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())