fix(ci): harden promote workflow — env-isolated untrusted input, main-ancestry check, concurrency guard

Addresses the AI review findings on PR #33:
- severe: the workflow_dispatch sha input was interpolated directly into
  the run script (shell injection with REGISTRY_TOKEN + prod webhook in
  scope). It now reaches the shell only via env, is rejected if it
  starts with '-', and is resolved locally with git rev-parse.
- minor: the resolved sha must be a 40-hex ancestor of origin/main —
  non-main refs are refused explicitly instead of implicitly.
- minor: a prod-promotion concurrency group serialises promotions
  (cancel-in-progress: false).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0146VHeLAWjDVE2B5uU67jCB

.gitea/workflows/deploy.yml

@@ -1,4 +1,4 @@
-name: Deploy (staging -> E2E gate -> production)
+name: Stage (build -> staging -> E2E gate)
 
 on:
   push:
@@ -193,48 +193,5 @@ jobs:
         env:
           BASE_URL: ${{ secrets.STAGING_BASE_URL }}
 
-  promote-prod:
-    name: Promote to Production
-    runs-on: ubuntu-latest
-    needs: [e2e-staging]
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Log in to Gitea Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ gitea.actor }}
-          password: ${{ secrets.REGISTRY_TOKEN }}
-
-      - name: Retag verified images as prod
-        run: |
-          SHORT_SHA="sha-$(echo "${{ gitea.sha }}" | cut -c1-7)"
-          for IMAGE in \
-            "${REGISTRY}/${BACKEND_IMAGE_NAME}" \
-            "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \
-            "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do
-            # Keep a rollback pointer before moving :prod
-            docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true
-            docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}"
-            echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod"
-          done
-
-      - name: Trigger production stack update
-        run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}"
-
-      - name: Wait for production to become healthy
-        run: |
-          echo "Polling ${PROD_BASE_URL} for up to 5 minutes..."
-          for i in $(seq 1 60); do
-            if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then
-              echo "Production is up (attempt $i)"
-              exit 0
-            fi
-            sleep 5
-          done
-          echo "Production did not become healthy in time" >&2
-          exit 1
-        env:
-          PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }}
+# Production deployment is a second, manual approval: see promote.yml
+# ("Promote to Production (manual)") and docs/DEPLOY.md.

.gitea/workflows/promote.yml

@@ -0,0 +1,126 @@
+name: Promote to Production (manual)
+
+# Second approval gate of the deploy model: run this workflow from the
+# Actions UI after testing the feature on staging. It refuses commits
+# whose staging E2E gate is not green. See docs/DEPLOY.md.
+
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: >-
+          Commit SHA on main to promote (full or >=7 chars).
+          Leave empty to promote the latest main commit.
+        required: false
+        default: ""
+
+# Only one promotion at a time; never cancel an in-flight promotion.
+concurrency:
+  group: prod-promotion
+  cancel-in-progress: false
+
+env:
+  REGISTRY: privaterepo.sitaru.org
+  BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend
+  FRONTEND_IMAGE_NAME: ${{ gitea.repository }}-frontend
+  PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline
+
+jobs:
+  promote-prod:
+    name: Promote approved commit to Production
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository (full history for ancestry check)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Resolve and validate target SHA
+        id: resolve
+        # SECURITY: the dispatch input is untrusted — it reaches the shell
+        # only via env (never spliced into `run:` with ${{ }}) and is only
+        # used as a quoted argument. The resolved value is validated as a
+        # 40-hex sha and required to be an ancestor of main before any
+        # later step interpolates it.
+        env:
+          SHA_INPUT: ${{ gitea.event.inputs.sha }}
+        run: |
+          set -euo pipefail
+          case "$SHA_INPUT" in
+            -*) echo "REFUSED: SHA input may not start with '-'." >&2; exit 1 ;;
+          esac
+          if [ -z "$SHA_INPUT" ]; then
+            SHA_INPUT="$(git rev-parse origin/main)"
+          fi
+          FULL_SHA=$(git rev-parse --verify --quiet "${SHA_INPUT}^{commit}") || {
+            echo "REFUSED: not a commit in this repository." >&2
+            exit 1
+          }
+          echo "$FULL_SHA" | grep -Eq '^[0-9a-f]{40}$'
+          if ! git merge-base --is-ancestor "$FULL_SHA" origin/main; then
+            echo "REFUSED: $FULL_SHA is not on main — only main commits are promotable." >&2
+            exit 1
+          fi
+          SHORT_SHA="sha-$(echo "$FULL_SHA" | cut -c1-7)"
+          echo "full=$FULL_SHA" >> "$GITHUB_OUTPUT"
+          echo "short=$SHORT_SHA" >> "$GITHUB_OUTPUT"
+          echo "Promoting $FULL_SHA (images tagged $SHORT_SHA)"
+
+      - name: Verify the staging E2E gate passed for this commit
+        run: |
+          STATUS_JSON=$(curl -fsS \
+            -H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
+            "https://${REGISTRY}/api/v1/repos/${{ gitea.repository }}/commits/${{ steps.resolve.outputs.full }}/status")
+          echo "$STATUS_JSON" | python3 -c "
+          import json, sys
+          d = json.load(sys.stdin)
+          ok = [s for s in d.get('statuses', [])
+                if 'E2E Journeys against Staging' in s.get('context', '')
+                and s.get('status') == 'success']
+          if not ok:
+              print('REFUSED: no successful \"E2E Journeys against Staging\" status on this commit.')
+              print('Contexts found:', [s.get('context') for s in d.get('statuses', [])])
+              sys.exit(1)
+          print('E2E gate verified green for this commit.')
+          "
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ gitea.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Retag approved images as prod (keeping rollback pointer)
+        run: |
+          SHORT_SHA="${{ steps.resolve.outputs.short }}"
+          for IMAGE in \
+            "${REGISTRY}/${BACKEND_IMAGE_NAME}" \
+            "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \
+            "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do
+            # Keep a rollback pointer before moving :prod
+            docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true
+            docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}"
+            echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod"
+          done
+
+      - name: Trigger production stack update
+        run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}"
+
+      - name: Wait for production to become healthy
+        run: |
+          echo "Polling ${PROD_BASE_URL} for up to 5 minutes..."
+          for i in $(seq 1 60); do
+            if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then
+              echo "Production is up (attempt $i)"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "Production did not become healthy in time" >&2
+          exit 1
+        env:
+          PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }}

backend/gias_codes.py

@@ -78,6 +78,7 @@ OFFICIAL_SIXTH_FORM: dict[int, str] = {
     0: "Not applicable",
     1: "Has a sixth form",
     2: "Does not have a sixth form",
+    9: "",
 }
 
 RELIGIOUS_CHARACTER: dict[int, str] = {
@@ -128,12 +129,14 @@ RELIGIOUS_CHARACTER: dict[int, str] = {
     47: "Reformed Baptist",
     48: "Roman Catholic/Anglican",
     49: "Sunni Deobandi",
+    99: "",
 }
 
 ADMISSIONS_POLICY: dict[int, str] = {
     0: "Not applicable",
     2: "Selective",
     4: "Non-selective",
+    9: "",
 }
 
 

backend/tests/test_gias_codes.py

@@ -81,3 +81,15 @@ def test_seed_matches_dictionaries():
         for row in csv.DictReader(fh):
             seed[row["field"]][int(row["code"])] = row["name"]
     assert seed == fields
+
+
+def test_blank_name_sentinel_codes_map_to_empty_string():
+    """GIAS carries codes whose (name) column is blank — e.g. ReligiousCharacter
+    99 (~4k schools) and AdmissionsPolicy 9 (~5.6k schools). The old name
+    pipeline served these as empty strings; the dictionaries must reproduce
+    that ("" is falsy, so UI tag heuristics stay silent) rather than letting
+    them hit the "Unknown (<code>)" path meant for genuinely new codes."""
+    assert RELIGIOUS_CHARACTER[99] == ""
+    assert ADMISSIONS_POLICY[9] == ""
+    assert translate(99, RELIGIOUS_CHARACTER) == ""
+    assert translate(9, ADMISSIONS_POLICY) == ""

claude.md

@@ -112,11 +112,15 @@ Full details in `docs/DEPLOY.md`. The short version:
 - **Never push to `main` directly.** Work on a feature branch and open a PR;
   branch protection requires the PR checks (typecheck, tests, builds, AI review)
   to pass before merge.
-- Merging to `main` deploys automatically: images are built once, deployed to
-  the **staging** Portainer stack, verified by the Playwright journeys in
-  `e2e/`, and only then retagged `:prod` and rolled out to production.
+- Merging to `main` deploys automatically **to staging only**: images are
+  built once, deployed to the staging Portainer stack, and verified by the
+  Playwright journeys in `e2e/`. Production is a second, manual approval:
+  the "Promote to Production (manual)" workflow in Gitea Actions, run after
+  testing the feature on staging. It refuses commits whose staging E2E gate
+  isn't green. Never trigger it yourself — promotion is the human's call.
 - If you change user-facing behaviour, update or extend the `e2e/` journey
-  tests in the same PR — they are the promotion gate.
+  tests in the same PR — they gate whether staging is fit for human testing
+  and whether a commit is promotable.
 
 ## Recent Changes
 

docs/DEPLOY.md

@@ -1,41 +1,61 @@
 # SDLC & Deployment Pipeline
 
-SchoolCompare uses a fully automated staging → production pipeline on Gitea
-Actions. AI writes the code on feature branches; the pipeline verifies every
-change on a staging environment before promoting the exact same images to
-production. Human input is directional only: feature requests, PR review if
-desired, and intervention when a gate fails.
+SchoolCompare uses a two-stage deploy model on Gitea Actions with two human
+approvals. AI writes the code on feature branches; the first approval merges
+the PR, which deploys to staging and runs the E2E gate; the second approval —
+after manual testing on staging — promotes the exact same images to
+production via a manual workflow.
 
 ## The flow
 
 ```
 feature branch (AI-authored)
-   │  PR to main
+   │  PR to main                                    ← approval #1
    ▼
 PR checks (.gitea/workflows/pr-checks.yml)
    typecheck + unit tests + backend smoke + image builds (no push)
    + Claude code review posted as a PR comment (severe findings fail the check)
    │  merge (branch protection requires green checks)
    ▼
-Deploy pipeline (.gitea/workflows/deploy.yml)
+Stage pipeline (.gitea/workflows/deploy.yml) — automatic
    1. build & push images        → tags sha-<sha>, staging
    2. staging Portainer webhook  → wait for staging health
-   3. Playwright E2E journeys against staging
-   4. retag sha-<sha> → :prod    (same bytes — build once, promote the image)
+   3. Playwright E2E journeys against staging      ← gate before human testing
+   ▼
+Manual testing on staging (stx.schoolcompare.co.uk)
+   │  Actions → "Promote to Production (manual)"   ← approval #2
+   ▼
+Promote pipeline (.gitea/workflows/promote.yml) — manual dispatch
+   1. resolve target sha (input, or latest main if empty)
+   2. REFUSE unless that commit's "E2E Journeys against Staging" status is green
+   3. retag sha-<sha> → :prod    (same bytes — build once, promote the image)
       previous :prod saved as :prod-previous
-   5. prod Portainer webhook     → wait for prod health
+   4. prod Portainer webhook     → wait for prod health
 ```
 
 Key principle: **build once, promote the exact image**. Production pins `:prod`,
-which only moves after the E2E gate passes on staging. Nothing tags `:latest`
+which only moves when a human runs the promote workflow — and the workflow
+only accepts commits that passed the staging E2E gate. Nothing tags `:latest`
 anymore.
 
 ## Branch & PR workflow
 
 - `main` is protected: no direct pushes, PRs require green status checks.
 - All work (human or AI) happens on feature branches → PR to `main`.
-- Merging to `main` **is** the release action. If staging or the E2E gate
-  fails, production is untouched.
+- Merging to `main` releases **to staging only**. Production moves only on
+  the second approval. If staging or the E2E gate fails, fix forward —
+  production is untouched either way.
+
+## Promotion granularity
+
+Staging always runs the latest `main`. Promoting approves a *state of main*,
+not a single PR — if two PRs merged since the last promotion, they ship
+together. Test staging accordingly. To promote an older state, pass its
+commit SHA to the promote workflow (its images must still exist in the
+registry).
+
+Staging quirk for manual testing: external `/api` is broken at the staging
+proxy — exercise API endpoints from the host, not via the public staging URL.
 
 ## Environments
 
@@ -92,8 +112,9 @@ fail the E2E gate. That's the point: staging absorbs the risk.
 
 ## Rollback
 
-Every promotion first re-points `:prod-previous` at the outgoing `:prod`.
-To roll back:
+Re-run "Promote to Production (manual)" with the SHA of the last good commit
+(fastest, fully gated), or manually re-point the tags — every promotion first
+saves the outgoing `:prod` as `:prod-previous`:
 
 ```bash
 for img in backend frontend pipeline; do

docs/superpowers/plans/2026-07-13-staged-prod-promotion.md

@@ -0,0 +1,275 @@
+# Staged Production Promotion (Manual Gate) Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Merging a PR deploys to staging only; production deployment requires a second, explicit human approval after manual testing on staging.
+
+**Architecture:** Split the existing single `deploy.yml` pipeline in two. The push-to-main workflow keeps build → staging deploy → e2e gate and **stops there**. A new `promote.yml` runs only on `workflow_dispatch` (the "Run workflow" button in Gitea's Actions UI, supported on this server — Gitea 1.26.4): it verifies the chosen commit passed the staging e2e gate, retags its `:sha-*` images to `:prod` (keeping `:prod-previous` for rollback), and triggers the Portainer prod webhook. Promotion granularity is a main-branch commit: staging always runs the latest main, so you approve a *state of main*, not an individual PR.
+
+**Tech Stack:** Gitea Actions (1.26.4), Docker buildx imagetools, Portainer webhooks, Gitea commit-status API.
+
+## Global Constraints
+
+- **Never push to `main` directly** — this change itself goes through a PR (`chore/staged-prod-promotion` branch).
+- Existing image tagging scheme is unchanged: `type=sha` (e.g. `sha-6f925ab`) + `:staging`; promotion still retags `:sha-*` → `:prod` with `:prod-previous` kept as the rollback pointer.
+- The e2e journeys remain a **hard gate before human testing** (a red staging never reaches the promote button) and the promote workflow must refuse to promote a commit whose staging e2e did not succeed.
+- Secrets already exist and are reused: `REGISTRY_TOKEN` (also a Gitea API token), `PORTAINER_STAGING_WEBHOOK`, `PORTAINER_PROD_WEBHOOK`, `STAGING_BASE_URL`, `PROD_BASE_URL`.
+- Staging quirk (memory): external `/api` is broken at the staging proxy — manual API testing happens from the host, not through stx.schoolcompare.co.uk; note it in the runbook, don't try to fix it in this plan.
+
+## Considered approaches (context for the reviewer)
+
+1. **Manual `workflow_dispatch` promote workflow (chosen).** Native on Gitea 1.26; the second approval is clicking "Run workflow" (or one API call) after testing staging. Least machinery, auditable via the Actions run history.
+2. *Tag-driven promotion* (`push: tags: promote-*`): works on any Gitea version; approval = pushing a tag. Slightly more scriptable, less discoverable; kept as documented fallback only.
+3. *GitOps `production` branch + promotion PR:* approval literally reuses the PR-review UI, but adds a second long-lived branch to keep in sync — too much ceremony for a solo project. Rejected.
+
+---
+
+### Task 0: Branch
+
+- [ ] `git checkout main && git pull && git checkout -b chore/staged-prod-promotion`
+
+---
+
+### Task 1: Stop the push-to-main workflow after the e2e gate
+
+**Files:**
+- Modify: `.gitea/workflows/deploy.yml`
+
+**Interfaces:**
+- Produces: images tagged `:sha-<short>` + `:staging` (unchanged), a green `E2E Journeys against Staging` commit status that Task 2's promote workflow checks by name. **Do not rename the `e2e-staging` job's `name:` without updating Task 2's status check.**
+
+- [ ] **Step 1: Remove the auto-promotion**
+
+In `.gitea/workflows/deploy.yml`:
+1. Change line 1 to: `name: Stage (build -> staging -> E2E gate)`
+2. Delete the entire `promote-prod` job (lines 196–240 in the current file: from `  promote-prod:` to the end of the file).
+3. Leave `build-*`, `deploy-staging`, and `e2e-staging` untouched.
+
+- [ ] **Step 2: Sanity-check the YAML**
+
+Run: `python3 -c "import yaml; yaml.safe_load(open('.gitea/workflows/deploy.yml')); print('yaml ok')"`
+Expected: `yaml ok`
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add .gitea/workflows/deploy.yml
+git commit -m "ci: stop deploy pipeline at staging; production promotion becomes manual"
+```
+
+---
+
+### Task 2: Manual promote workflow
+
+**Files:**
+- Create: `.gitea/workflows/promote.yml`
+
+**Interfaces:**
+- Consumes: `:sha-<short>` images built by deploy.yml; the `E2E Journeys against Staging` commit status.
+- Produces: `:prod` and `:prod-previous` tags; prod stack update.
+
+- [ ] **Step 1: Write the workflow**
+
+```yaml
+name: Promote to Production (manual)
+
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: >-
+          Commit SHA on main to promote (full or >=7 chars).
+          Leave empty to promote the latest main commit.
+        required: false
+        default: ""
+
+env:
+  REGISTRY: privaterepo.sitaru.org
+  BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend
+  FRONTEND_IMAGE_NAME: ${{ gitea.repository }}-frontend
+  PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline
+
+jobs:
+  promote-prod:
+    name: Promote approved commit to Production
+    runs-on: ubuntu-latest
+    steps:
+      - name: Resolve target SHA
+        id: resolve
+        run: |
+          SHA_INPUT="${{ gitea.event.inputs.sha }}"
+          if [ -z "$SHA_INPUT" ]; then
+            SHA_INPUT="${{ gitea.sha }}"
+          fi
+          # Normalise to the full sha via the API so short inputs work
+          FULL_SHA=$(curl -fsS \
+            -H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
+            "https://${REGISTRY}/api/v1/repos/${{ gitea.repository }}/git/commits/${SHA_INPUT}" \
+            | python3 -c "import json,sys; print(json.load(sys.stdin)['sha'])")
+          SHORT_SHA="sha-$(echo "$FULL_SHA" | cut -c1-7)"
+          echo "full=$FULL_SHA" >> "$GITHUB_OUTPUT"
+          echo "short=$SHORT_SHA" >> "$GITHUB_OUTPUT"
+          echo "Promoting $FULL_SHA (images tagged $SHORT_SHA)"
+
+      - name: Verify the staging E2E gate passed for this commit
+        run: |
+          STATUS_JSON=$(curl -fsS \
+            -H "Authorization: token ${{ secrets.REGISTRY_TOKEN }}" \
+            "https://${REGISTRY}/api/v1/repos/${{ gitea.repository }}/commits/${{ steps.resolve.outputs.full }}/status")
+          echo "$STATUS_JSON" | python3 -c "
+          import json, sys
+          d = json.load(sys.stdin)
+          ok = [s for s in d.get('statuses', [])
+                if 'E2E Journeys against Staging' in s.get('context', '')
+                and s.get('status') == 'success']
+          if not ok:
+              print('REFUSED: no successful \"E2E Journeys against Staging\" status on this commit.')
+              print('Contexts found:', [s.get('context') for s in d.get('statuses', [])])
+              sys.exit(1)
+          print('E2E gate verified green for this commit.')
+          "
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ gitea.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Retag approved images as prod (keeping rollback pointer)
+        run: |
+          SHORT_SHA="${{ steps.resolve.outputs.short }}"
+          for IMAGE in \
+            "${REGISTRY}/${BACKEND_IMAGE_NAME}" \
+            "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \
+            "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do
+            docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true
+            docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}"
+            echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod"
+          done
+
+      - name: Trigger production stack update
+        run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}"
+
+      - name: Wait for production to become healthy
+        run: |
+          echo "Polling ${PROD_BASE_URL} for up to 5 minutes..."
+          for i in $(seq 1 60); do
+            if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then
+              echo "Production is up (attempt $i)"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "Production did not become healthy in time" >&2
+          exit 1
+        env:
+          PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }}
+```
+
+Implementation notes for the engineer:
+- Gitea Actions uses the GitHub-compatible `$GITHUB_OUTPUT` file for step outputs; if the runner image doesn't populate it, fall back to `$GITEA_OUTPUT` (check the runner's docs/output at first run).
+- The retag step is copied verbatim from the old `promote-prod` job except the SHA comes from the resolved input instead of `gitea.sha` — behaviour for the default (empty input on latest main) is identical to before.
+- If `docker buildx imagetools create` fails with "not found" for `${IMAGE}:${SHORT_SHA}`, the chosen commit predates the registry's retention or never built — the error message is the desired behaviour (refuse loudly).
+
+- [ ] **Step 2: YAML sanity check**
+
+Run: `python3 -c "import yaml; yaml.safe_load(open('.gitea/workflows/promote.yml')); print('yaml ok')"`
+Expected: `yaml ok`
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add .gitea/workflows/promote.yml
+git commit -m "ci: manual production promotion workflow with e2e-gate verification"
+```
+
+---
+
+### Task 3: Documentation — deploy model + runbook
+
+**Files:**
+- Modify: `docs/DEPLOY.md`
+- Modify: `claude.md` (the SDLC section)
+
+- [ ] **Step 1: Rewrite the flow description in `docs/DEPLOY.md`**
+
+Replace the staging→prod description with the new model (adapt to the file's existing structure; the substance to convey):
+
+```markdown
+## Deploy model
+
+1. **PR → main (first approval).** Branch-protected merge; PR checks
+   (typecheck, tests, builds, AI review) must pass.
+2. **Merge → staging (automatic).** Images are built once and tagged
+   `sha-<short>` + `staging`; the staging stack updates; Playwright
+   journeys in `e2e/` run against staging. A red e2e run means staging
+   is not fit for testing — fix forward before considering promotion.
+3. **Manual testing on staging.** stx.schoolcompare.co.uk. Note:
+   external `/api` is broken at the staging proxy — exercise API
+   endpoints from the host.
+4. **Promote → production (second approval).** Actions → "Promote to
+   Production (manual)" → Run workflow. Leave the SHA empty to promote
+   the latest main, or paste a specific commit SHA. The workflow
+   refuses commits whose staging e2e gate is not green, retags the
+   images `:prod` (keeping `:prod-previous`), and updates the prod
+   stack.
+
+### Promotion granularity
+
+Staging always runs the latest `main`. Promoting approves a *state of
+main*, not a single PR — if two PRs merged since the last promotion,
+they ship together. Test staging accordingly.
+
+### Rollback
+
+Re-run "Promote to Production (manual)" with the SHA of the last good
+commit (or retag manually: `docker buildx imagetools create -t
+<image>:prod <image>:prod-previous` for each of the three images, then
+POST the prod Portainer webhook).
+```
+
+- [ ] **Step 2: Update the SDLC bullet in `claude.md`**
+
+Replace the sentence "Merging to `main` deploys automatically: … retagged `:prod` and rolled out to production." with:
+
+```markdown
+- Merging to `main` deploys automatically **to staging only**: images
+  are built once, deployed to the staging Portainer stack, and verified
+  by the Playwright journeys in `e2e/`. Production is a second, manual
+  approval: the "Promote to Production (manual)" workflow in Gitea
+  Actions, run after testing the feature on staging. It refuses commits
+  whose staging e2e gate isn't green.
+```
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add docs/DEPLOY.md claude.md
+git commit -m "docs: two-stage deploy model (staging auto, production manual)"
+```
+
+---
+
+### Task 4: PR + live validation
+
+- [ ] **Step 1: Push and open the PR** (Gitea API with credential-helper basic auth, as usual). PR body: the new model in three lines, the rollback recipe, and a warning that between merging this PR and its first promotion run, production receives no deployments (expected).
+
+- [ ] **Step 2: Validate after merge (human-in-the-loop):**
+1. Merge this PR → confirm the `Stage (build -> staging -> E2E gate)` run goes green and **no** production deployment happens (prod image digest unchanged: `docker buildx imagetools inspect <image>:prod` before/after, or check the Portainer prod stack's last-update time).
+2. Test something trivial on staging.
+3. Run "Promote to Production (manual)" with the SHA empty → confirm e2e verification passes, retag happens, prod becomes healthy.
+4. Negative test: run the promote workflow with a garbage SHA (e.g. `deadbeef1`) → confirm it fails at resolve/verify without touching `:prod`.
+
+- [ ] **Step 3: Update the ledger/memory** with the new deploy model so future sessions stop assuming auto-promotion.
+
+---
+
+## Out of scope / future options
+
+- Notifications when staging is ready for testing (Gitea can email on workflow completion; a webhook to ntfy/Matrix could be added later).
+- Restricting who can run the promote workflow: Gitea 1.26 runs `workflow_dispatch` with the permissions of the dispatching user; for a solo repo this is already effectively restricted.
+- The tag-driven fallback (`on: push: tags: promote-*`) if `workflow_dispatch` ever proves unreliable on the runner.

pipeline/dags/school_data_pipeline.py

@@ -38,6 +38,31 @@ default_args = {
     "retry_delay": timedelta(minutes=5),
 }
 
+# The backend caches the marts DataFrame at startup; after any rebuild the
+# cache must be invalidated or the API serves stale (or empty) data until the
+# container restarts.
+INVALIDATE_CACHE_CMD = """
+set -e
+BACKEND_URL="${BACKEND_URL:-http://backend:80}"
+ADMIN_KEY="${ADMIN_API_KEY:-changeme}"
+
+echo "Calling $BACKEND_URL/api/admin/reload ..."
+
+response=$(curl -s -o /tmp/reload_response.json -w "%{http_code}" \\
+    --connect-timeout 10 --max-time 120 \\
+    -X POST "$BACKEND_URL/api/admin/reload" \\
+    -H "X-API-Key: $ADMIN_KEY" \\
+    -H "Content-Type: application/json")
+
+echo "HTTP status: $response"
+cat /tmp/reload_response.json
+
+if [ "$response" != "200" ]; then
+    echo "ERROR: backend cache reload failed (HTTP $response)"
+    exit 1
+fi
+"""
+
 
 # ── Daily DAG (GIAS + downstream) ──────────────────────────────────────
 
@@ -91,7 +116,12 @@ print(f'Validation passed: {{count}} GIAS rows')
         bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py",
     )
 
-    extract_group >> validate_raw >> dbt_build >> sync_typesense
+    invalidate_cache = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_group >> validate_raw >> dbt_build >> sync_typesense >> invalidate_cache
 
 
 # ── Monthly DAG (Ofsted) ───────────────────────────────────────────────
@@ -121,7 +151,12 @@ with DAG(
         bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py",
     )
 
-    extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted
+    invalidate_cache_ofsted = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_ofsted >> dbt_build_ofsted >> sync_typesense_ofsted >> invalidate_cache_ofsted
 
 
 # ── Annual DAG (EES: KS2, KS4, Census, Admissions) ───────────────────
@@ -153,7 +188,12 @@ with DAG(
         bash_command=f"cd {PIPELINE_DIR} && python scripts/sync_typesense.py",
     )
 
-    extract_ees_group >> dbt_build_ees >> sync_typesense_ees
+    invalidate_cache_ees = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_ees_group >> dbt_build_ees >> sync_typesense_ees >> invalidate_cache_ees
 
 
 # ── Annual DAG (IDACI Deprivation) ────────────────────────────────────
@@ -178,4 +218,9 @@ with DAG(
         bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_idaci+ fact_deprivation+",
     )
 
-    extract_idaci >> dbt_build_idaci
+    invalidate_cache_idaci = BashOperator(
+        task_id="invalidate_cache",
+        bash_command=INVALIDATE_CACHE_CMD,
+    )
+
+    extract_idaci >> dbt_build_idaci >> invalidate_cache_idaci

pipeline/scripts/generate_gias_codes.py

@@ -94,13 +94,23 @@ def main() -> None:
     for code_col, name_col, dict_name, field_key in FIELDS:
         pairs = (
             df[[code_col, name_col]]
-            .loc[lambda d: (d[code_col] != "") & (d[name_col] != "")]
+            .loc[lambda d: d[code_col] != ""]
             .drop_duplicates()
         )
-        mapping = sorted((int(c), n) for c, n in pairs.itertuples(index=False))
-        dupes = len(mapping) - len({c for c, _ in mapping})
-        if dupes:
-            sys.exit(f"{code_col}: {dupes} codes map to multiple names — investigate before generating")
+        by_code: dict[int, set] = {}
+        for c, n in pairs.itertuples(index=False):
+            by_code.setdefault(int(c), set()).add(n)
+        mapping = []
+        for code, names in sorted(by_code.items()):
+            named = sorted(n for n in names if n != "")
+            if len(named) > 1:
+                sys.exit(f"{code_col}: code {code} maps to multiple names {named} — investigate before generating")
+            # Codes that only ever appear with a blank (name) are GIAS
+            # "not recorded" sentinels (e.g. ReligiousCharacter 99,
+            # AdmissionsPolicy 9). Map them to "" so the API serves the same
+            # empty string the old name pipeline did — the "Unknown (<code>)"
+            # path is reserved for genuinely new codes.
+            mapping.append((code, named[0] if named else ""))
         lines = [f"{dict_name}: dict[int, str] = {{"]
         for code, name in mapping:
             escaped = name.replace('"', '\\"')

pipeline/scripts/gias_codes.py

@@ -78,6 +78,7 @@ OFFICIAL_SIXTH_FORM: dict[int, str] = {
     0: "Not applicable",
     1: "Has a sixth form",
     2: "Does not have a sixth form",
+    9: "",
 }
 
 RELIGIOUS_CHARACTER: dict[int, str] = {
@@ -128,12 +129,14 @@ RELIGIOUS_CHARACTER: dict[int, str] = {
     47: "Reformed Baptist",
     48: "Roman Catholic/Anglican",
     49: "Sunni Deobandi",
+    99: "",
 }
 
 ADMISSIONS_POLICY: dict[int, str] = {
     0: "Not applicable",
     2: "Selective",
     4: "Non-selective",
+    9: "",
 }
 
 

pipeline/transform/models/marts/_marts_schema.yml

@@ -42,12 +42,12 @@ models:
         tests:
           - accepted_values:
               severity: warn
-              values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
+              values: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 99]
       - name: admissions_policy_code
         tests:
           - accepted_values:
               severity: warn
-              values: [0, 2, 4]
+              values: [0, 2, 4, 9]
 
   - name: dim_location
     description: School location dimension with PostGIS geometry

pipeline/transform/seeds/gias_code_names.csv

@@ -53,6 +53,7 @@ phase_of_education,7,All-through
 official_sixth_form,0,Not applicable
 official_sixth_form,1,Has a sixth form
 official_sixth_form,2,Does not have a sixth form
+official_sixth_form,9,
 religious_character,0,Does not apply
 religious_character,2,Church of England
 religious_character,3,Roman Catholic
@@ -100,6 +101,8 @@ religious_character,46,Protestant/Evangelical
 religious_character,47,Reformed Baptist
 religious_character,48,Roman Catholic/Anglican
 religious_character,49,Sunni Deobandi
+religious_character,99,
 admissions_policy,0,Not applicable
 admissions_policy,2,Selective
 admissions_policy,4,Non-selective
+admissions_policy,9,