docs(about-blog): design for a named author, an About page and a Payload blog

The site reads as synthetic because nobody is accountable for the
numbers, no editorial judgement is visible, and the voice is
institutional third person. This designs the fix: a named author
(first name, photo, explicitly not an education expert), a coded
/about page, and a blog backed by Payload CMS running inside the
existing Next app.

Also fills a hole in the SEO programme, which has eight workstreams
and no E-E-A-T or authorship signal on a YMYL corpus.

Records two collisions found while designing, both of which fail
badly if missed: Payload's default /api route fights the existing
FastAPI catch-all proxy, and withPayload() is ESM-only so
next.config.js — which carries staging's noindex header — has to
become next.config.mjs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FT1Ls4GbgLDXoQX7NAuHGT

docker-compose.portainer.staging.yml

@@ -18,7 +18,10 @@
 #   TYPESENSE_SEARCH_KEY   — Typesense search-only key (exposed to frontend)
 #   UNLEASH_URL            — http://<unleash-ip>:4242/api (empty = all flags off)
 #   UNLEASH_API_TOKEN      — Unleash *client* token, environment: development
-#   AIRFLOW_ADMIN_USER     — Airflow admin username (password auto-generated, see api-server logs)
+#   AIRFLOW_ADMIN_USER     — Airflow admin username (default: admin)
+#   AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server
+#                            refuses to start without it, rather than falling
+#                            back to a generated one that changes on restart.
 #   STAGING_DB_IP          — macvlan IP for staging Postgres (default 10.0.1.190)
 #   STAGING_FRONTEND_IP    — macvlan IP for staging frontend (default 10.0.1.151)
 
@@ -124,7 +127,23 @@ services:
   airflow-api-server:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:staging
     container_name: sc_staging_airflow_api
-    command: airflow api-server --port 8080
+    # The simple auth manager generates a random password on first start and
+    # writes it to a file, so every container restart invalidates the last one.
+    # Writing the file ourselves from an environment variable makes the login
+    # deterministic. Airflow does not generate anything when the file exists.
+    #
+    # Built with python rather than echo/printf so a password containing quotes,
+    # backslashes or spaces is escaped correctly by json.dumps. An unset
+    # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling
+    # back to a generated password would silently undo the point of this.
+    command:
+      - bash
+      - -c
+      - |
+        set -euo pipefail
+        mkdir -p /opt/airflow
+        python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))"
+        exec airflow api-server --port 8080
     ports:
       - "8081:8080"
     environment:
@@ -136,6 +155,8 @@ services:
       AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-staging-airflow-jwt-secret-key-long-enough-for-sha512"
       AIRFLOW__API_AUTH__JWT_ISSUER: airflow
       AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json
+      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?set AIRFLOW_ADMIN_PASSWORD in the Portainer stack environment}
       AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
       PG_HOST: sc_database
       PG_PORT: "5432"

docker-compose.portainer.yml

@@ -9,7 +9,10 @@
 #   TYPESENSE_SEARCH_KEY   — Typesense search-only key (exposed to frontend)
 #   UNLEASH_URL            — http://<unleash-ip>:4242/api (empty = all flags off)
 #   UNLEASH_API_TOKEN      — Unleash *client* token, environment: production
-#   AIRFLOW_ADMIN_USER     — Airflow admin username (password auto-generated, see api-server logs)
+#   AIRFLOW_ADMIN_USER     — Airflow admin username (default: admin)
+#   AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server
+#                            refuses to start without it, rather than falling
+#                            back to a generated one that changes on restart.
 
 services:
 
@@ -113,7 +116,23 @@ services:
   airflow-api-server:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod
     container_name: schoolcompare_airflow_api
-    command: airflow api-server --port 8080
+    # The simple auth manager generates a random password on first start and
+    # writes it to a file, so every container restart invalidates the last one.
+    # Writing the file ourselves from an environment variable makes the login
+    # deterministic. Airflow does not generate anything when the file exists.
+    #
+    # Built with python rather than echo/printf so a password containing quotes,
+    # backslashes or spaces is escaped correctly by json.dumps. An unset
+    # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling
+    # back to a generated password would silently undo the point of this.
+    command:
+      - bash
+      - -c
+      - |
+        set -euo pipefail
+        mkdir -p /opt/airflow
+        python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))"
+        exec airflow api-server --port 8080
     ports:
       - "8080:8080"
     environment:
@@ -125,6 +144,8 @@ services:
       AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512"
       AIRFLOW__API_AUTH__JWT_ISSUER: airflow
       AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json
+      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?set AIRFLOW_ADMIN_PASSWORD in the Portainer stack environment}
       AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs
       PG_HOST: sc_database
       PG_PORT: "5432"

docker-compose.yml

@@ -105,7 +105,23 @@ services:
   airflow-api-server:
     image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest
     container_name: schoolcompare_airflow_api
-    command: airflow api-server --port 8080
+    # The simple auth manager generates a random password on first start and
+    # writes it to a file, so every container restart invalidates the last one.
+    # Writing the file ourselves from an environment variable makes the login
+    # deterministic. Airflow does not generate anything when the file exists.
+    #
+    # Built with python rather than echo/printf so a password containing quotes,
+    # backslashes or spaces is escaped correctly by json.dumps. An unset
+    # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling
+    # back to a generated password would silently undo the point of this.
+    command:
+      - bash
+      - -c
+      - |
+        set -euo pipefail
+        mkdir -p /opt/airflow
+        python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))"
+        exec airflow api-server --port 8080
     ports:
       - "8080:8080"
     environment: &airflow-env
@@ -117,6 +133,8 @@ services:
       AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512"
       AIRFLOW__API_AUTH__JWT_ISSUER: airflow
       AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "admin:admin"
+      AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json
+      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-admin}
       PG_HOST: db
       PG_PORT: "5432"
       PG_USER: schoolcompare

docs/DEPLOY.md

@@ -98,6 +98,12 @@ fail the E2E gate. That's the point: staging absorbs the risk.
    pr-checks status checks (frontend, backend, builds, ai-review) to pass.
 5. **Bootstrap staging data via Airflow** (no prod dump — staging populates
    itself from source, exercising the pipeline image end-to-end):
+   - Set `AIRFLOW_ADMIN_PASSWORD` in the stack environment first. The
+     api-server refuses to start without it. Airflow's simple auth manager
+     otherwise generates a password on first start and writes it to a file, so
+     the login changes every time the container restarts; the stack writes that
+     file itself from this variable instead. `AIRFLOW_ADMIN_USER` defaults to
+     `admin`.
    - Open the staging Airflow UI (`http://<host>:8081`) and trigger, in order:
      `school_data_daily`, `school_data_monthly_ofsted`, then the manual-schedule
      `school_data_annual_ees` and `school_data_annual_idaci`.

docs/superpowers/specs/2026-09-02-about-and-blog-design.md

@@ -0,0 +1,341 @@
+# Giving schoolcompare a human author: an About page and a blog
+
+**Date:** 2026-09-02
+**Status:** Design — awaiting review
+**Scope:** A named author for the site, an `/about` page, and a Payload-CMS-backed
+blog at `/blog`.
+
+## Why
+
+The site reads as synthetic. Not because of its tone, but because of three
+specific absences:
+
+1. **Nobody is accountable for the numbers.** There is no author, no statement
+   of why the site exists, and no one who can be wrong. The only human trace on
+   the entire site is `contact@schoolcompare.co.uk` in the footer.
+2. **No visible judgement.** Every figure is presented as though it fell out of
+   a machine. Hundreds of editorial decisions went into this codebase — which
+   metrics to show, when a benchmark is invalid, what to suppress — and not one
+   of them is visible to a reader. `isSpecialSchool()` silently drops the
+   England comparison for special schools and PRUs because that comparison is
+   meaningless; nowhere does the site *say* so.
+3. **The voice is institutional third person.** "schoolcompare brings it all
+   into one place." "Built for parents, governors, journalists." That is
+   brochure register, and it is precisely the register that machine-generated
+   content defaults to.
+
+There is a second, independent reason. The SEO programme
+(`2026-08-20-seo-programme-design.md`) defines eight workstreams and none of
+them address E-E-A-T or authorship. School performance data is YMYL territory;
+an anonymous site republishing DfE figures has no authorship signal at all. This
+work fills that hole, and the blog gives W6 (explainer content) somewhere to
+live.
+
+### The failure mode to avoid
+
+The standard fix — a stock photo and "Hi, I'm Tudor, and I'm passionate about
+education!" — reads as *more* synthetic than the current coldness. Manufactured
+warmth is a stronger machine-tell than plain institutional voice. Everything
+here has to be specific, occasionally awkward, and willing to be unflattering,
+or it makes the problem worse.
+
+## Positioning
+
+The author is **Tudor**: first name only, real photograph, no surname, no
+employer named.
+
+The credibility claim is deliberately **not** educational expertise. The About
+page states plainly: *"I'm not an education expert."* Authority comes from two
+things that are actually true:
+
+- **Experience.** A parent going through primary admissions in south-west London
+  right now. Google's E-E-A-T leads with Experience, and lived experience of the
+  thing is exactly what the DfE's own service lacks.
+- **Method.** Every number's provenance is stated, so a reader can check the
+  site rather than trust it.
+
+This is more durable than borrowed expertise: it cannot be undermined by someone
+noticing the author has no teaching qualification.
+
+**Consequence for the design.** A `Person` entity with no surname is a weak
+search signal and cannot be corroborated off-site. The credibility load
+therefore shifts onto the methodology being visibly rigorous. That is a design
+constraint, not a caveat — it is why the About page carries a substantial
+"how this is built and where it can be wrong" section rather than a short bio.
+
+### Voice rules
+
+Applied to About and every post. Recorded here so the voice does not drift.
+
+- First person singular. "I built", not "we provide".
+- Concrete over general. "when we were looking at schools in Wandsworth" beats
+  any amount of stated warmth.
+- State limits before someone else finds them. Every post that presents a
+  metric says what it does not show.
+- No mission statements, no "passionate about", no invented team.
+- Short sentences. The existing code comments in this repo are already written
+  this way; the prose should match.
+
+## Scope
+
+**In:**
+
+- `/about` — a coded page (not CMS-managed).
+- `/blog` and `/blog/[slug]` — Payload-backed, with an index and post pages.
+- Payload CMS installed into the existing Next application.
+- Footer and navigation links to both.
+- `Person`, `Organization`, `BlogPosting`, `BreadcrumbList` JSON-LD.
+- RSS feed and sitemap integration.
+- One first post, so the blog does not launch empty.
+
+**Out (deliberately):**
+
+- Rewriting existing homepage/how-it-works copy into first person. Worth doing,
+  but it would double the review surface of this PR. Separate change.
+- In-product signed notes on school pages (the "distributed humanity" idea).
+  Revisit once About and the blog exist.
+- Comments, newsletter, author accounts beyond one.
+- A team page. There is no team.
+
+## Architecture
+
+### Topology
+
+Payload 3 installs **into the existing Next application** and serves `/admin`
+from the same container. One image, one deploy, no new service. This is
+Payload 3's native model and it makes on-demand revalidation trivial, because
+the CMS hooks run in the same process as the Next cache.
+
+Accepted costs: the public site's image now carries Payload, so a CMS security
+patch redeploys the whole site; and the image grows substantially.
+
+### Two collisions that must be handled
+
+**1. `/api` is already taken.** `app/api/[...path]/route.ts` is a catch-all that
+proxies `/api/*` to FastAPI at runtime. Payload's default API route is also
+`/api`. Left alone, these fight, and the failure is not clean — the catch-all
+would swallow Payload's admin API calls and forward them to FastAPI.
+
+Payload's API route is therefore remapped:
+
+```ts
+routes: { api: '/cms-api', admin: '/admin' }
+```
+
+with its route group at `app/(payload)/cms-api/[...slug]/route.ts`. The
+`/cms-api` prefix must also be added to the FastAPI proxy's excluded-paths list
+as a defensive second line.
+
+**2. `next.config.js` is CommonJS.** Payload's `withPayload()` wrapper is ESM
+only. The config must become `next.config.mjs`, converting `module.exports` to
+`export default` and wrapping the export. All existing content — the standalone
+output, `outputFileTracingIncludes`, the staging `X-Robots-Tag` header block,
+the CSP — carries over unchanged. This is mechanical but it touches the file
+that controls staging's noindex, so it needs care and an explicit test.
+
+### Database
+
+Payload uses the existing `sc_database` Postgres instance, in its **own
+`payload` schema**:
+
+```ts
+db: postgresAdapter({
+  pool: { connectionString: process.env.DATABASE_URL },
+  schemaName: 'payload',
+})
+```
+
+The frontend container is already on the `backend` Docker network, so it can
+reach `sc_database:5432` with no networking change. It needs a new
+`DATABASE_URL` environment variable.
+
+Schema isolation is not cosmetic. `public` currently holds the application
+tables and Airflow's metadata, and `scripts/migrate_csv_to_db.py --drop` exists
+to drop and reimport. Blog content living in its own schema means no data
+pipeline operation can destroy it. **Before implementation, confirm that
+`--drop` is schema-scoped and cannot reach `payload`.**
+
+Putting CMS tables in this instance is consistent with existing practice —
+Airflow already stores its metadata there.
+
+### Migrations
+
+Payload's Postgres adapter auto-pushes schema in development and requires
+explicit migrations in production. Use `prodMigrations`, which runs pending
+migrations during server initialisation:
+
+```ts
+db: postgresAdapter({ /* ... */, prodMigrations: migrations })
+```
+
+This is preferred over a one-shot init container (the `airflow-init` pattern)
+because the app is a single long-running process and there is no ordering
+problem to solve. Migration files are generated with `payload migrate:create`
+and committed, so schema changes travel through the same PR and staging gate as
+code.
+
+### Media
+
+Uploads go to a Docker named volume, consistent with `postgres_data`,
+`typesense_data` and `airflow_logs`.
+
+- `staticDir` must be an **absolute** path in Payload 3: `/app/media`.
+- The container runs as `nextjs` (uid 1001). The Dockerfile must
+  `mkdir -p /app/media && chown nextjs:nodejs /app/media` **before** the volume
+  is mounted, or Docker will create the mountpoint root-owned and every upload
+  will fail with EACCES.
+- `sharp` moves from `devDependencies` to `dependencies` — Payload needs it at
+  runtime to generate `imageSizes`.
+- The volume must be added to the backup routine alongside Postgres. A blog
+  post's images are not reproducible from the pipeline.
+
+### Rendering
+
+**Constraint:** CI builds the image with no database reachable. Blog pages
+therefore cannot use build-time `generateStaticParams` — that would either fail
+the build or bake in an empty post list.
+
+Instead: ISR. Post and index pages declare a `revalidate` window and render on
+first request, with Payload `afterChange` / `afterDelete` hooks calling
+`revalidatePath('/blog')` and `revalidatePath('/blog/' + slug)` for immediate
+publication. Because Payload runs in the same process, the hook calls
+`revalidatePath` from `next/cache` directly — no webhook, no shared secret.
+
+The ISR cache lives on container disk and is cleared by a redeploy. For a
+single container serving a handful of posts this is fine.
+
+### Collections
+
+- **`posts`** — `title`, `slug`, `publishedAt`, `excerpt`, `heroImage`
+  (relation to `media`), `content` (Lexical rich text), `seo` group
+  (`metaTitle`, `metaDescription`), `_status` (drafts enabled).
+- **`media`** — upload collection, `alt` required, `imageSizes` for thumbnail
+  and hero widths, public read access.
+- **`users`** — Payload's auth collection. One account. Public creation
+  disabled.
+
+Drafts are enabled so posts can be written over several sittings and previewed
+before publication.
+
+**Payload Blocks** are how posts embed live product components — a real trend
+chart or comparison table inside a post, rendered from live data rather than
+screenshotted. This is the main thing the CMS has to earn back against
+file-based MDX, and it directly serves the goal: showing judgement in context.
+Ship with one block (a callout/aside for "what this number doesn't tell you");
+add a live-chart block once a post needs it.
+
+### Security
+
+`/admin` is the first authenticated surface on this site. Public, hardened:
+
+- `PAYLOAD_SECRET` — long, random, set in the Portainer stack environment, never
+  committed. The same variable must exist in staging with a *different* value.
+- Strong unique password on the single admin account.
+- Login rate limiting via Payload's `maxLoginAttempts` / `lockTime`.
+- `X-Robots-Tag: noindex, nofollow` on `/admin/*` and `/cms-api/*`, and a
+  `robots.ts` disallow. The admin panel must never be indexed.
+- Public user creation disabled; no open registration.
+- Verify the existing CSP `frame-ancestors` directive does not break the admin
+  panel.
+
+Residual risk, accepted: a future Payload authentication CVE is live against the
+public internet. Mitigation is prompt patching, which the staging→prod pipeline
+already supports. If this becomes uncomfortable, restricting `/admin` at the
+proxy to LAN/VPN is a one-line change later.
+
+Staging note: staging runs the same image on `stx.`, so it gets its own admin
+panel and its own database. It must have its own `PAYLOAD_SECRET` and its own
+credentials — never production's.
+
+## Deployment changes
+
+- `nextjs-app/Dockerfile` — create and chown `/app/media`; ensure Payload's
+  admin bundle and `sharp` survive standalone output file tracing.
+- `docker-compose.portainer.yml` and the staging equivalent — add
+  `DATABASE_URL` and `PAYLOAD_SECRET` to the `frontend` service, add a
+  `payload_media` volume mounted at `/app/media`, and add
+  `depends_on: sc_database`.
+- Document both new environment variables in the compose header comment block,
+  which is where this stack records its configuration.
+
+## SEO
+
+- `Person` (Tudor, with photo) and `Organization` JSON-LD on `/about`.
+- `BlogPosting` + `BreadcrumbList` on post pages, with `author` referencing the
+  same `Person`.
+- Canonical URLs on `/blog` and every post.
+- Posts and `/about` added to the existing sitemap (`app/sitemap.xml/route.ts`
+  and `app/sitemaps/[...parts]`). Post URLs come from Payload at request time.
+- RSS feed at `/blog/rss.xml`.
+- Footer links to both pages, under a new "About" column.
+
+**Navigation is deliberately left alone.** `Navigation.tsx` renders a bottom tab
+bar on mobile that already carries four items (Search, Compare, Rankings,
+Admissions). A fifth tab makes each one cramped at 320px, and About and Blog are
+both lower-intent than any of the four. Both live in the footer; About
+additionally gets a byline link from every post, which is where a reader who
+cares actually asks the question. Revisit only if analytics show people hunting
+for it.
+
+## Testing
+
+Unit (Jest):
+
+- Post rendering, including a post with no hero image and one with no excerpt.
+- Slug generation and collision handling.
+- JSON-LD shape for `BlogPosting` and `Person`.
+- The `next.config.mjs` conversion preserves the staging `X-Robots-Tag` rule —
+  this guards the riskiest mechanical change in the plan.
+
+E2E (Playwright, `e2e/`, required by CLAUDE.md for user-facing change):
+
+- `/about` renders, shows the author name and photo, and is reachable from the
+  footer and nav.
+- `/blog` lists at least one post; clicking through reaches the post.
+- A post page renders title, date, body and byline.
+- `/admin` responds with `noindex` and does not leak a stack trace when
+  unauthenticated.
+
+Note the known constraint: new journeys cannot be proven in PR checks, because
+the staging E2E gate runs post-merge.
+
+## Risks
+
+| Risk | Mitigation |
+|---|---|
+| `next.config.mjs` conversion silently drops the staging noindex header, making staging a crawlable duplicate | Unit test asserting the header rule; verify on staging before promotion |
+| Payload API route collides with the FastAPI `/api` proxy | Remap to `/cms-api`; add to the proxy's exclusion list |
+| Media volume mounts root-owned; all uploads fail with EACCES | `mkdir`+`chown` in the Dockerfile before the mount; test an upload on staging |
+| Build fails or bakes empty content because CI has no DB | No build-time DB access; ISR only |
+| A pipeline `--drop` destroys blog content | Separate `payload` schema; verify `--drop` blast radius before building |
+| Media volume not backed up; images unrecoverable | Add `payload_media` to the backup routine |
+| Payload auth CVE exposed publicly | Prompt patching; proxy restriction available as a fallback |
+| Blog launches empty or goes stale | Ship with one post; cadence is explicitly "a few times a year", so no cadence is promised anywhere on the page — no dates implying a schedule |
+
+## Sequence
+
+Each step is independently reviewable and mergeable.
+
+1. **Payload foundation** — install, `next.config.mjs` conversion, `payload`
+   schema, `/cms-api` remap, `users` collection, `/admin` hardening, compose and
+   Dockerfile changes. No public-facing change yet. Verify on staging that the
+   site is unchanged and `/admin` works.
+2. **`/about`** — coded page, photo, `Person`/`Organization` JSON-LD, footer and
+   nav links, e2e journey. Independently valuable and does not depend on the
+   blog.
+3. **Blog** — `posts` and `media` collections, `/blog` index and post pages, ISR
+   plus revalidation hooks, RSS, sitemap, structured data, e2e journeys.
+4. **First post** — written in the admin panel, published through the normal
+   flow, proving the whole path end to end.
+
+Step 1 carries all the infrastructure risk and none of the visible benefit, so
+it should be verified on staging carefully before step 2 starts.
+
+## Dependencies on Tudor
+
+- **A photograph.** Blocks step 2. Nothing else in the plan is blocked by it.
+- **The first post's subject.** Blocks step 4 only. Suggested: what school
+  performance data cannot tell you — it demonstrates judgement, is genuinely
+  useful, and is the kind of thing an anonymous or machine-written site will not
+  publish.
+- **Confirmation** that `scripts/migrate_csv_to_db.py --drop` is schema-scoped.

pipeline/Dockerfile

@@ -18,6 +18,7 @@ COPY plugins/ plugins/
 RUN pip install --no-cache-dir \
     ./plugins/extractors/tap-uk-gias \
     ./plugins/extractors/tap-uk-ees \
+    ./plugins/extractors/tap-uk-ees-destinations \
     ./plugins/extractors/tap-uk-ofsted \
     ./plugins/extractors/tap-uk-fbit \
     ./plugins/extractors/tap-uk-idaci

pipeline/dags/school_data_pipeline.py

@@ -190,7 +190,7 @@ with DAG(
 
     dbt_build_ees = BashOperator(
         task_id="dbt_build",
-        bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ees_ks2+ stg_legacy_ks2+ stg_ees_ks4+ stg_legacy_ks4+ stg_ees_census+ stg_ees_admissions+ stg_ees_ks2_national+ stg_ees_ks4_national+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+",
+        bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ees_ks2+ stg_legacy_ks2+ stg_ees_ks4+ stg_legacy_ks4+ stg_ees_census+ stg_ees_admissions+ stg_ees_ks2_national+ stg_ees_ks4_national+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+ stg_ees_ks4_destinations_national+ stg_ees_ks5_destinations_national+",
     )
 
     sync_typesense_ees = BashOperator(

pipeline/plugins/extractors/tap-uk-ees-destinations/tap_uk_ees_destinations/tap.py

@@ -188,21 +188,25 @@ class DestinationsStream(Stream):
     _national_pinned: list[str] = []
     _indicators: dict[str, str] = {}
 
-    schema = th.PropertiesList(
-        th.Property("urn", th.StringType),
+    # School rows and the England reference are DIFFERENT GRAINS, so they are
+    # different streams. Carrying both in one table meant a null `urn` inside
+    # the primary key, which target-postgres turns into a NOT NULL constraint:
+    # the first national row killed the loader mid-run, and the tap saw only a
+    # BrokenPipeError on its stdout.
+    _MEASURE_PROPERTIES = (
         th.Property("time_period", th.StringType),
         th.Property("pupil_group", th.StringType),
         th.Property("destination_measure", th.StringType),
         th.Property("cohort_pupils", th.StringType),
         th.Property("pupils_raw", th.StringType),
         th.Property("percentage_raw", th.StringType),
-    ).to_dict()
+    )
+    _MEASURE_KEYS = ["time_period", "pupil_group", "destination_measure"]
 
-    primary_keys = ["urn", "time_period", "pupil_group", "destination_measure"]
     replication_key = None
 
     def _query(self, period: str, pinned: list[str], keep_level: str,
-               urn_by_location: dict[str, str]):
+               urn_by_location: dict[str, str]):  # noqa: D401
         """Page one period of one geographic level, yielding Singer records."""
         criteria = [
             {"filters": {"in": list(self._destination_slugs)}},
@@ -248,19 +252,51 @@ class DestinationsStream(Stream):
             "%s: %d school locations, %d time periods",
             self.name, len(urn_by_location), len(periods),
         )
-
-        # Two queries per period, because school rows and the England reference
-        # need different establishment pins — see KS4_NATIONAL_PINNED. The API
-        # cannot filter by geographic level, so each pass keeps its own and
-        # discards the local-authority, district, regional and constituency
-        # rows that come with them.
         for period in periods:
-            yield from self._query(period, self._pinned, "SCH", urn_by_location)
-            yield from self._query(period, self._national_pinned, "NAT", urn_by_location)
+            yield from self._query(
+                period, self._pinned_for_level, self._level, urn_by_location,
+            )
 
 
-class KS4DestinationsStream(DestinationsStream):
-    name = "ees_ks4_destinations"
+class SchoolDestinationsStream(DestinationsStream):
+    """School-level rows. `urn` is part of the key and is never null."""
+
+    _level = "SCH"
+
+    @property
+    def _pinned_for_level(self):
+        return self._pinned
+
+    schema = th.PropertiesList(
+        th.Property("urn", th.StringType),
+        *DestinationsStream._MEASURE_PROPERTIES,
+    ).to_dict()
+
+    primary_keys = ["urn", *DestinationsStream._MEASURE_KEYS]
+
+
+class NationalDestinationsStream(DestinationsStream):
+    """The England reference. No `urn` column at all — a school identifier that
+    is always null is not a column, it is a grain mismatch."""
+
+    _level = "NAT"
+
+    @property
+    def _pinned_for_level(self):
+        return self._national_pinned
+
+    schema = th.PropertiesList(*DestinationsStream._MEASURE_PROPERTIES).to_dict()
+
+    primary_keys = list(DestinationsStream._MEASURE_KEYS)
+
+    def post_process(self, row, context=None):
+        # row_to_record emits urn=None for national rows; drop the key rather
+        # than ship a column that is null in every row.
+        row.pop("urn", None)
+        return row
+
+
+class _KS4Config(DestinationsStream):
     _dataset_id = KS4_DATASET
     _destination_slugs = KS4_DESTINATION_SLUGS
     _pupil_group_slugs = KS4_PUPIL_GROUP_SLUGS
@@ -269,8 +305,7 @@ class KS4DestinationsStream(DestinationsStream):
     _indicators = KS4_INDICATORS
 
 
-class KS5DestinationsStream(DestinationsStream):
-    name = "ees_ks5_destinations"
+class _KS5Config(DestinationsStream):
     _dataset_id = KS5_DATASET
     _destination_slugs = KS5_DESTINATION_SLUGS
     _pupil_group_slugs = KS5_PUPIL_GROUP_SLUGS
@@ -279,12 +314,33 @@ class KS5DestinationsStream(DestinationsStream):
     _indicators = KS5_INDICATORS
 
 
+class KS4DestinationsStream(SchoolDestinationsStream, _KS4Config):
+    name = "ees_ks4_destinations"
+
+
+class KS5DestinationsStream(SchoolDestinationsStream, _KS5Config):
+    name = "ees_ks5_destinations"
+
+
+class KS4NationalDestinationsStream(NationalDestinationsStream, _KS4Config):
+    name = "ees_ks4_destinations_national"
+
+
+class KS5NationalDestinationsStream(NationalDestinationsStream, _KS5Config):
+    name = "ees_ks5_destinations_national"
+
+
 class TapUKEESDestinations(Tap):
     name = "tap-uk-ees-destinations"
     config_jsonschema = th.PropertiesList().to_dict()
 
     def discover_streams(self):
-        return [KS4DestinationsStream(self), KS5DestinationsStream(self)]
+        return [
+            KS4DestinationsStream(self),
+            KS5DestinationsStream(self),
+            KS4NationalDestinationsStream(self),
+            KS5NationalDestinationsStream(self),
+        ]
 
 
 if __name__ == "__main__":

pipeline/plugins/extractors/tap-uk-ees-destinations/tests/test_tap.py

@@ -132,3 +132,68 @@ def test_establishment_dimensions_are_not_pinned():
     establishment_totals = {"4369U", "rgHcN", "EfHQq", "S4ROV"}
     assert not set(KS4_PINNED) & establishment_totals
     assert not set(KS5_PINNED) & establishment_totals
+
+
+# ── Grain separation ────────────────────────────────────────────────────────
+#
+# School rows and the England reference were originally one stream with a
+# nullable `urn` in the primary key. target-postgres turns primary_keys into a
+# NOT NULL constraint, so the first national row killed the loader mid-run and
+# the tap saw only a BrokenPipeError on its stdout — a symptom several frames
+# away from the cause.
+
+def _streams():
+    from tap_uk_ees_destinations.tap import TapUKEESDestinations
+    return TapUKEESDestinations(config={}, validate_config=False).discover_streams()
+
+
+def test_no_stream_has_a_nullable_primary_key_column():
+    for stream in _streams():
+        props = stream.schema["properties"]
+        for key in stream.primary_keys:
+            assert key in props, f"{stream.name}: key {key} is not in the schema"
+        if "urn" in stream.primary_keys:
+            assert stream._level == "SCH", (
+                f"{stream.name} keys on urn but does not emit school rows"
+            )
+
+
+def test_national_streams_carry_no_urn_column_at_all():
+    for stream in _streams():
+        if not stream.name.endswith("_national"):
+            continue
+        assert "urn" not in stream.schema["properties"], (
+            "a school identifier that is null in every row is a grain "
+            "mismatch, not a column"
+        )
+        assert "urn" not in stream.primary_keys
+
+
+def test_national_post_process_drops_the_null_urn():
+    from tap_uk_ees_destinations.tap import KS4NationalDestinationsStream, TapUKEESDestinations
+    tap = TapUKEESDestinations(config={}, validate_config=False)
+    stream = KS4NationalDestinationsStream(tap)
+    row = {"urn": None, "time_period": "202223", "pupil_group": "all",
+           "destination_measure": "school_sixth_form", "cohort_pupils": "1",
+           "pupils_raw": "1", "percentage_raw": "1"}
+    assert "urn" not in stream.post_process(dict(row))
+
+
+def test_school_and_national_streams_exist_for_both_phases():
+    names = {s.name for s in _streams()}
+    assert names == {
+        "ees_ks4_destinations", "ees_ks5_destinations",
+        "ees_ks4_destinations_national", "ees_ks5_destinations_national",
+    }
+
+
+def test_each_stream_queries_its_own_geographic_level_with_its_own_pins():
+    """The national pass needs establishment pinned to Total; the school pass
+    must not pin it at all, or every school returns zero rows."""
+    for stream in _streams():
+        if stream.name.endswith("_national"):
+            assert stream._level == "NAT"
+            assert stream._pinned_for_level is stream._national_pinned
+        else:
+            assert stream._level == "SCH"
+            assert stream._pinned_for_level is stream._pinned

pipeline/transform/models/marts/fact_destination_national.sql

@@ -22,8 +22,7 @@ select
     pupils,
     percentage,
     status
-from {{ ref('stg_ees_ks4_destinations') }}
-where urn is null
+from {{ ref('stg_ees_ks4_destinations_national') }}
 
 union all
 
@@ -36,5 +35,4 @@ select
     pupils,
     percentage,
     status
-from {{ ref('stg_ees_ks5_destinations') }}
-where urn is null
+from {{ ref('stg_ees_ks5_destinations_national') }}

pipeline/transform/models/staging/_stg_sources.yml

@@ -47,6 +47,17 @@ sources:
           16-18 study leavers destinations. Same grain, same suppression
           caveat, and only institutions with post-16 provision appear.
 
+      - name: ees_ks4_destinations_national
+        description: >
+          England KS4 destination measures by pupil group. A separate table
+          from ees_ks4_destinations because it is a separate grain — no school,
+          so no urn column. Same 'c' suppression caveat.
+
+      - name: ees_ks5_destinations_national
+        description: >
+          England 16-18 destination measures by pupil group. Same grain and
+          caveat as ees_ks4_destinations_national.
+
       - name: ees_ks4_performance
         description: KS4 performance tables (long format — one row per school × breakdown × sex)
 

pipeline/transform/models/staging/stg_ees_ks4_destinations.sql

@@ -1,7 +1,9 @@
 {{ config(materialized='table') }}
 
--- Staging model: KS4 leavers destinations, school level plus the England
--- reference (which carries a null urn).
+-- Staging model: KS4 leavers destinations, school level.
+--
+-- School rows only. The England reference is a different grain and lives in
+-- stg_ees_ks4_destinations_national.
 --
 -- DELIBERATELY DOES NOT USE safe_numeric. That macro maps every EES sentinel
 -- (z, c, x, q, u) to NULL, which is right for attainment — there, "suppressed"
@@ -14,13 +16,12 @@
 
 with source as (
     select * from {{ source('raw', 'ees_ks4_destinations') }}
-    -- National rows carry a null urn and feed fact_destination_national.
-    where (urn is null or urn = '' or urn ~ '^[0-9]+$')
+    where urn ~ '^[0-9]+$'
       and time_period ~ '^[0-9]+$'
 )
 
 select
-    case when urn ~ '^[0-9]+$' then cast(trim(urn) as integer) end as urn,
+    cast(trim(urn) as integer) as urn,
     cast(trim(time_period) as integer)                            as year,
     trim(pupil_group)                                             as pupil_group,
     trim(destination_measure)                                     as destination_measure,

pipeline/transform/models/staging/stg_ees_ks4_destinations_national.sql

@@ -0,0 +1,38 @@
+{{ config(materialized='table') }}
+
+-- Staging model: England KS4 destination measures — the national
+-- reference the school sections compare against.
+--
+-- A separate model because it is a separate grain: there is no school here, and
+-- carrying these rows in the school table meant a null urn inside the primary
+-- key, which the Postgres loader rejects.
+--
+-- DELIBERATELY DOES NOT USE safe_numeric, for the same reason as the school
+-- model: 'suppressed' and 'not applicable' are different claims.
+
+with source as (
+    select * from {{ source('raw', 'ees_ks4_destinations_national') }}
+    where time_period ~ '^[0-9]+$'
+)
+
+select
+    cast(trim(time_period) as integer) as year,
+    trim(pupil_group)                  as pupil_group,
+    trim(destination_measure)          as destination_measure,
+
+    case when cohort_pupils ~ '^[0-9]+$'
+         then cast(cohort_pupils as integer) end as cohort_pupils,
+
+    case when pupils_raw ~ '^[0-9]+$'
+         then cast(pupils_raw as integer) end as pupils,
+
+    case when percentage_raw ~ '^-?[0-9]+(\.[0-9]+)?$'
+         then cast(percentage_raw as numeric) end as percentage,
+
+    case
+        when pupils_raw ~ '^[0-9]+$'        then 'published'
+        when lower(trim(pupils_raw)) = 'c'  then 'suppressed'
+        else 'not_applicable'
+    end as status
+
+from source

pipeline/transform/models/staging/stg_ees_ks5_destinations.sql

@@ -1,8 +1,10 @@
 {{ config(materialized='table') }}
 
--- Staging model: 16-18 study leavers destinations, institution level plus the
--- England reference (which carries a null urn). Only sixth forms and colleges
--- appear here, so a secondary with no post-16 provision has no rows at all.
+-- Staging model: 16-18 study leavers destinations, institution level.
+--
+-- Only sixth forms and colleges appear, so a secondary with no post-16
+-- provision has no rows at all. The England reference is a different grain and
+-- lives in stg_ees_ks5_destinations_national.
 --
 -- DELIBERATELY DOES NOT USE safe_numeric. That macro maps every EES sentinel
 -- (z, c, x, q, u) to NULL, which is right for attainment — there, "suppressed"
@@ -15,13 +17,12 @@
 
 with source as (
     select * from {{ source('raw', 'ees_ks5_destinations') }}
-    -- National rows carry a null urn and feed fact_destination_national.
-    where (urn is null or urn = '' or urn ~ '^[0-9]+$')
+    where urn ~ '^[0-9]+$'
       and time_period ~ '^[0-9]+$'
 )
 
 select
-    case when urn ~ '^[0-9]+$' then cast(trim(urn) as integer) end as urn,
+    cast(trim(urn) as integer) as urn,
     cast(trim(time_period) as integer)                            as year,
     trim(pupil_group)                                             as pupil_group,
     trim(destination_measure)                                     as destination_measure,

pipeline/transform/models/staging/stg_ees_ks5_destinations_national.sql

@@ -0,0 +1,38 @@
+{{ config(materialized='table') }}
+
+-- Staging model: England KS5 destination measures — the national
+-- reference the school sections compare against.
+--
+-- A separate model because it is a separate grain: there is no school here, and
+-- carrying these rows in the school table meant a null urn inside the primary
+-- key, which the Postgres loader rejects.
+--
+-- DELIBERATELY DOES NOT USE safe_numeric, for the same reason as the school
+-- model: 'suppressed' and 'not applicable' are different claims.
+
+with source as (
+    select * from {{ source('raw', 'ees_ks5_destinations_national') }}
+    where time_period ~ '^[0-9]+$'
+)
+
+select
+    cast(trim(time_period) as integer) as year,
+    trim(pupil_group)                  as pupil_group,
+    trim(destination_measure)          as destination_measure,
+
+    case when cohort_pupils ~ '^[0-9]+$'
+         then cast(cohort_pupils as integer) end as cohort_pupils,
+
+    case when pupils_raw ~ '^[0-9]+$'
+         then cast(pupils_raw as integer) end as pupils,
+
+    case when percentage_raw ~ '^-?[0-9]+(\.[0-9]+)?$'
+         then cast(percentage_raw as numeric) end as percentage,
+
+    case
+        when pupils_raw ~ '^[0-9]+$'        then 'published'
+        when lower(trim(pupils_raw)) = 'c'  then 'suppressed'
+        else 'not_applicable'
+    end as status
+
+from source