diff --git a/claude.md b/claude.md index 3d05e66..057a630 100644 --- a/claude.md +++ b/claude.md @@ -23,6 +23,46 @@ Key files: - `backend/data_loader.py` - Data queries, geocoding, legacy DataFrame compatibility - `backend/schemas.py` - Column mappings, metric definitions, LA code mappings +### Content / CMS (Payload) + +Payload CMS runs **inside** the Next.js app — one image, one container, no +separate service. It powers `/blog`; `/about` is a plain coded page. + +- **Admin panel:** `/admin`. The only authenticated surface on the site. + `noindex` via both `robots.txt` and `X-Robots-Tag`. +- **CMS API:** `/cms-api`, **not** `/api`. `/api/*` is a catch-all proxy to + FastAPI (`app/(frontend)/api/[...path]`) which would silently swallow every + admin call and forward it to the backend. Mount points are defined once in + `lib/payloadRoutes.ts`. +- **Database:** the existing Postgres, in its own `payload` schema, so no + pipeline operation on `public` — including + `scripts/migrate_csv_to_db.py --drop` — can reach blog content. +- **Uploads:** the `payload_media` Docker volume at `/app/media`. Not + reproducible from the pipeline; must be backed up. +- **New env vars:** `DATABASE_URL` and `PAYLOAD_SECRET` on the frontend service. + Staging must use a different `PAYLOAD_SECRET` from production. +- Publishing workflow and house style: `nextjs-app/docs/PUBLISHING.md`. + +### Two route groups + +`nextjs-app/app/` has no root `layout.tsx`. It cannot: Payload's admin panel +ships its own root layout rendering ``/``, and Next permits +multiple root layouts only when no `app/layout.tsx` exists. + +- `app/(frontend)/` — the site. Its `layout.tsx` is the site's root layout. +- `app/(payload)/` — the admin panel and `/cms-api`. + +Route groups are invisible to routing, so every public URL is unchanged. + +**The metadata file conventions stay at the `app/` root** — `robots.ts`, +`opengraph-image.tsx`, `icon.png`, `apple-icon.png`. Inside a route group Next +treats them as segment-scoped: it renames `/icon.png` to `/icon-.png` and +drops `/robots.txt` entirely. Route handlers are unaffected. + +The build must succeed with `DATABASE_URL` unset, because CI builds it that +way. Never call `getCachedPayload()` at module scope, and never add +`generateStaticParams` to a DB-backed route. + ### Frontend (Vanilla JS) - Single-page application with hash-based routing - Chart.js for data visualization diff --git a/docker-compose.portainer.staging.yml b/docker-compose.portainer.staging.yml index c08ae48..99cc608 100644 --- a/docker-compose.portainer.staging.yml +++ b/docker-compose.portainer.staging.yml @@ -18,6 +18,10 @@ # TYPESENSE_SEARCH_KEY — Typesense search-only key (exposed to frontend) # UNLEASH_URL — http://:4242/api (empty = all flags off) # UNLEASH_API_TOKEN — Unleash *client* token, environment: development +# PAYLOAD_SECRET — Payload CMS encryption secret. REQUIRED: long and +# random, and DIFFERENT from production's. Sharing +# it would let a staging session authenticate +# against production. # AIRFLOW_ADMIN_USER — Airflow admin username (default: admin) # AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server # refuses to start without it, rather than falling @@ -89,9 +93,20 @@ services: - FASTAPI_URL=http://backend:80/api - TYPESENSE_URL=http://typesense:8108 - TYPESENSE_API_KEY=${TYPESENSE_SEARCH_KEY:-changeme} + # Payload CMS runs inside this container, in the `payload` schema of the + # staging database. Staging has its own stack, its own Postgres and its + # own admin account — never production's. + - DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME} + - PAYLOAD_SECRET=${PAYLOAD_SECRET:?set PAYLOAD_SECRET in the staging Portainer stack environment} + volumes: + # Portainer prefixes volume names with the stack name, so this is + # automatically isolated from production's media. + - payload_media:/app/media depends_on: backend: condition: service_healthy + sc_database: + condition: service_healthy networks: backend: {} macvlan: @@ -242,3 +257,4 @@ volumes: typesense_data: airflow_logs: unleash_cache: + payload_media: diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml index 00e085e..fd2be50 100644 --- a/docker-compose.portainer.yml +++ b/docker-compose.portainer.yml @@ -9,6 +9,9 @@ # TYPESENSE_SEARCH_KEY — Typesense search-only key (exposed to frontend) # UNLEASH_URL — http://:4242/api (empty = all flags off) # UNLEASH_API_TOKEN — Unleash *client* token, environment: production +# PAYLOAD_SECRET — Payload CMS encryption secret. REQUIRED: long and +# random. Changing it invalidates every admin +# session. Staging MUST use a different value. # AIRFLOW_ADMIN_USER — Airflow admin username (default: admin) # AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server # refuses to start without it, rather than falling @@ -78,9 +81,21 @@ services: - FASTAPI_URL=http://backend:80/api - TYPESENSE_URL=http://typesense:8108 - TYPESENSE_API_KEY=${TYPESENSE_SEARCH_KEY:-changeme} + # Payload CMS runs inside this container. It reaches Postgres over the + # `backend` network and keeps its tables in the `payload` schema, so no + # pipeline operation on `public` can touch blog content. + - DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME} + # Same :? form as AIRFLOW_ADMIN_PASSWORD: refuse to start rather than + # boot with an empty secret and silently accept forged sessions. + - PAYLOAD_SECRET=${PAYLOAD_SECRET:?set PAYLOAD_SECRET in the Portainer stack environment} + volumes: + # Blog images. Not reproducible from the pipeline — must be backed up. + - payload_media:/app/media depends_on: backend: condition: service_healthy + sc_database: + condition: service_healthy networks: backend: {} macvlan: @@ -231,3 +246,4 @@ volumes: typesense_data: airflow_logs: unleash_cache: + payload_media: diff --git a/docs/superpowers/plans/2026-09-02-about-and-blog.md b/docs/superpowers/plans/2026-09-02-about-and-blog.md new file mode 100644 index 0000000..f1e8ff1 --- /dev/null +++ b/docs/superpowers/plans/2026-09-02-about-and-blog.md @@ -0,0 +1,2309 @@ +# About Page and Payload Blog Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give schoolcompare a named human author via an `/about` page and a +Payload-CMS-backed blog at `/blog`, both served from the existing Next.js app. + +**Architecture:** Payload 3 installs into the existing Next application and +serves `/admin` from the same container, storing content in its own `payload` +schema on the existing Postgres instance. Because Payload's admin panel needs +to be a *root* layout, the existing site routes first move into an +`app/(frontend)/` route group (URLs are unchanged — route groups are invisible +to routing). Blog pages use ISR with on-demand revalidation, because CI builds +the image with no database reachable. + +**Tech Stack:** Next.js 16 (App Router), React 19, TypeScript, Payload CMS 3 +(`@payloadcms/db-postgres`, `@payloadcms/next`, `@payloadcms/richtext-lexical`), +PostgreSQL, CSS Modules, Jest, Playwright, Docker. + +**Spec:** `docs/superpowers/specs/2026-09-02-about-and-blog-design.md` + +## Global Constraints + +These apply to every task. Copied from the spec. + +- **Author identity:** first name **Tudor** only. Never a surname. Never an + employer name. Never children's names. Location is "south-west London". +- **Credibility claim:** the About page must state plainly *"I'm not an + education expert."* Authority comes from lived experience (a parent going + through primary admissions) and from stated provenance of every number — + never from claimed educational qualifications. +- **Voice:** first person singular ("I built", never "we provide"); concrete + over general; state limits before someone else finds them; no mission + statements, no "passionate about", no invented team; short sentences. +- **Navigation is not touched.** `Navigation.tsx`'s mobile bottom tab bar + already carries four items; About and Blog are footer-only. Post pages link + to `/about` via the byline. +- **Design tokens only.** Use the existing CSS custom properties from + `app/globals.css` (`--bg-primary`, `--bg-card`, `--text-primary`, + `--text-secondary`, `--text-muted`, `--border`, `--brand`, `--brand-strong`, + `--brand-bg`, `--surface-sunken`). Never hardcode a hex value. Every new + surface must work in both light and dark themes. +- **Typography:** `var(--font-display)` (Manrope) for headings, + `var(--font-ui)` (Inter) for body. Never introduce a third family. +- **Absolute URLs** come from `absoluteUrl()` in `lib/site.ts`. Never hardcode + `https://www.schoolcompare.co.uk`. +- **No secrets committed.** `PAYLOAD_SECRET` and `DATABASE_URL` are supplied by + the environment. Staging must use different values from production. +- **CLAUDE.md rule:** user-facing behaviour changes ship with `e2e/` journey + updates in the same PR. +- **CLAUDE.md rule:** do not attempt to start a local server to test the + application — it does not work. Verification is via Jest, `npm run build`, + `npm run typecheck`, and the post-merge staging e2e gate. + +--- + +## Two structural findings that changed this plan + +Read these before starting. They were discovered while planning, not while +writing the spec, and they are the two places this work can go badly wrong. + +**1. The existing routes must move into `app/(frontend)/`.** +Next.js requires the *root* layout to render `` and ``. Payload's +admin panel ships its own root layout that also renders ``/``. With +today's `app/layout.tsx` in place, Payload's layout would nest inside the +site's — inheriting the nav, footer, fonts and `ComparisonProvider` — and +produce nested `` elements. The supported pattern is multiple root +layouts via route groups, which requires that **no `app/layout.tsx` exists**. +So every current route moves into `app/(frontend)/`. Route groups do not appear +in URLs, so every public URL is byte-identical afterwards. The move is +mechanical but it touches every route in the app, which is why it is its own +task (Task 2) with the full test suite as its gate. + +**2. The `/api` collision is avoided by the remap alone.** +The spec proposed also adding `/cms-api` to the FastAPI proxy's exclusion list +as a second line of defence. That is unnecessary and would be dead code: the +proxy is `app/api/[...path]/route.ts`, which only ever matches `/api/*`, and +`/cms-api` is a sibling path that never reaches it. Remapping Payload's API +route to `/cms-api` is sufficient and complete. Do not add the exclusion. + +--- + +## File Structure + +**Moved (Task 2)** — `app/*` → `app/(frontend)/*`, unchanged in content: +`layout.tsx`, `page.tsx`, `globals.css`, `rankings/`, `admissions/`, +`compare/`, `schools/`, `school/`, `api/`, `sitemaps/`, `sitemap.xml/`. + +**Deliberately NOT moved — they stay at the `app/` root:** `robots.ts`, +`opengraph-image.tsx`, `icon.png`, `apple-icon.png`. + +Next.js metadata file conventions only produce stable root URLs at the `app/` +root. Inside a route group they are treated as segment-scoped: verified during +execution, moving them into `(frontend)` renamed `/icon.png` to +`/icon-4usi79.png`, `/apple-icon.png` to `/apple-icon-4usi79.png`, +`/opengraph-image` to `/opengraph-image-4usi79`, and dropped `/robots.txt` +entirely. That would have broken the `/icon.png` cache-control rule and the +`outputFileTracingIncludes['/opengraph-image']` entry in `next.config.mjs`, +and silently removed the site's robots.txt. Route handlers (`sitemap.xml/`, +`sitemaps/`, `api/`) are unaffected and move normally. + +**Created:** + +| Path | Responsibility | +|---|---| +| `nextjs-app/next.config.mjs` | Replaces `next.config.js`; ESM so it can wrap `withPayload()` | +| `nextjs-app/payload.config.ts` | Payload's single source of truth: DB, routes, collections | +| `nextjs-app/collections/Users.ts` | Admin auth collection, lockout policy, closed registration | +| `nextjs-app/collections/Posts.ts` | Blog post schema, drafts, revalidation hooks | +| `nextjs-app/collections/Media.ts` | Upload collection writing to `/app/media` | +| `nextjs-app/blocks/Callout.ts` | The "what this number doesn't tell you" block | +| `nextjs-app/app/(payload)/**` | Generated Payload admin + `/cms-api` routes | +| `nextjs-app/migrations/**` | Committed Payload schema migrations | +| `nextjs-app/app/(frontend)/about/page.tsx` + `.module.css` | The About page | +| `nextjs-app/app/(frontend)/blog/page.tsx` + `.module.css` | Blog index | +| `nextjs-app/app/(frontend)/blog/[slug]/page.tsx` + `.module.css` | Post page | +| `nextjs-app/app/(frontend)/blog/rss.xml/route.ts` | RSS feed | +| `nextjs-app/app/(frontend)/content-sitemap.xml/route.ts` | Sitemap for Next-owned URLs | +| `nextjs-app/lib/payload.ts` | Cached `getPayload()` accessor | +| `nextjs-app/lib/jsonld.ts` | `Person` / `Organization` / `BlogPosting` builders | + +**Modified:** `package.json`, `tsconfig.json`, `Dockerfile`, +`docker-compose.portainer.yml`, `docker-compose.portainer.staging.yml`, +`components/Footer.tsx`, `__tests__/app/metadata.test.ts`, +`__tests__/api/proxyDenylist.test.ts`, `__tests__/app/placeMetadata.test.ts`, +`e2e/tests/journeys.spec.ts`. + +--- + +### Task 1: Convert `next.config.js` to ESM + +`withPayload()` is ESM-only, so the config must become `.mjs`. This file also +carries the rule that keeps staging out of Google's index — the highest-value +thing in the repo to break silently — so it is converted first, on its own, +behind a test. + +**Files:** +- Create: `nextjs-app/__tests__/app/nextConfig.test.ts` +- Create: `nextjs-app/next.config.mjs` +- Delete: `nextjs-app/next.config.js` + +**Interfaces:** +- Consumes: nothing. +- Produces: `next.config.mjs` default-exporting the Next config object. Task 3 + wraps this export in `withPayload()`. + +- [ ] **Step 1: Write the failing test** + +Create `nextjs-app/__tests__/app/nextConfig.test.ts`: + +```ts +/** + * next.config.mjs carries the staging noindex rule. Breaking it turns + * stx.schoolcompare.co.uk into a fully crawlable duplicate of production, + * and nothing else in the suite would notice. + */ +import nextConfig from '@/next.config.mjs'; + +describe('next.config.mjs', () => { + it('keeps the staging host out of the index', async () => { + const headers = await nextConfig.headers(); + const stagingRule = headers.find((rule) => + rule.has?.some( + (cond) => cond.type === 'host' && cond.value === 'stx.schoolcompare.co.uk', + ), + ); + expect(stagingRule).toBeDefined(); + expect(stagingRule.headers).toContainEqual({ + key: 'X-Robots-Tag', + value: 'noindex, nofollow', + }); + }); + + it('still emits standalone output for the Docker runner', () => { + expect(nextConfig.output).toBe('standalone'); + }); + + it('still traces the share-card fonts into the standalone bundle', () => { + expect(nextConfig.outputFileTracingIncludes['/opengraph-image']).toEqual([ + './assets/**', + ]); + }); + + it('still allows the analytics subdomain to frame the site', async () => { + const headers = await nextConfig.headers(); + const csp = headers + .flatMap((rule) => rule.headers) + .find((header) => header.key === 'Content-Security-Policy'); + expect(csp.value).toContain('https://analytics.schoolcompare.co.uk'); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cd nextjs-app && npm test -- __tests__/app/nextConfig.test.ts +``` + +Expected: FAIL — cannot resolve `@/next.config.mjs` (the file does not exist yet). + +If it instead fails with an ESM parse error after Step 3, add `'mjs'` to +`moduleFileExtensions` in `jest.config.js`: +```js +moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'mjs', 'json', 'node'], +``` + +- [ ] **Step 3: Convert the config** + +```bash +cd nextjs-app && git mv next.config.js next.config.mjs +``` + +Then edit `next.config.mjs`: change the final line from +`module.exports = nextConfig;` to `export default nextConfig;`. + +Change nothing else. Every comment block in that file documents a past +production incident — the baked-in `FASTAPI_URL`, the staging `X-Robots-Tag` +reasoning, the `/icon.png` cache 404 — and all of it must survive verbatim. + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +cd nextjs-app && npm test -- __tests__/app/nextConfig.test.ts +``` + +Expected: PASS, 4 tests. + +- [ ] **Step 5: Verify the whole suite and the build still pass** + +```bash +cd nextjs-app && npm test && npm run typecheck && npm run build +``` + +Expected: all green. `next/jest` resolves `.mjs` configs, so the existing +suite is unaffected. + +- [ ] **Step 6: Commit** + +```bash +git add nextjs-app/next.config.mjs nextjs-app/__tests__/app/nextConfig.test.ts nextjs-app/jest.config.js +git commit -m "build(next): convert the config to ESM so Payload can wrap it + +withPayload() is ESM-only. This file also carries the rule that keeps +staging out of the index, so the conversion goes in on its own behind a +test that asserts the rule survived." +``` + +--- + +### Task 2: Move site routes into an `app/(frontend)` route group + +This creates room for Payload's admin panel to be its own root layout. No URL +changes: route groups are invisible to routing. + +**Files:** +- Move: everything currently under `nextjs-app/app/` → `nextjs-app/app/(frontend)/` +- Modify: `nextjs-app/__tests__/app/metadata.test.ts:1-4` +- Modify: `nextjs-app/__tests__/app/placeMetadata.test.ts:1` +- Modify: `nextjs-app/__tests__/api/proxyDenylist.test.ts:11` + +**Interfaces:** +- Consumes: Task 1's `next.config.mjs`. +- Produces: no `app/layout.tsx` at the app root — the precondition Task 3 + requires. Site routes importable as `@/app/(frontend)//page`. + +- [ ] **Step 1: Record the baseline** + +```bash +cd nextjs-app && npm test 2>&1 | tail -5 +``` + +Write down the passing test count. It must be identical at Step 5. + +- [ ] **Step 2: Move every route into the group** + +```bash +cd nextjs-app/app && mkdir -p "(frontend)" +git mv layout.tsx page.tsx globals.css \ + rankings admissions compare schools school api sitemaps sitemap.xml \ + "(frontend)/" +``` + +`robots.ts`, `opengraph-image.tsx`, `icon.png` and `apple-icon.png` stay at the +`app/` root — see "Deliberately NOT moved" in the File Structure section above. +Moving them silently changes their URLs and drops robots.txt. + +Verify the split — `app/` should now contain `(frontend)` plus exactly the four +metadata conventions: + +```bash +cd /Users/tudor/projects/school_compare/nextjs-app && ls -A app +``` + +Expected: `(frontend)`, `apple-icon.png`, `icon.png`, `opengraph-image.tsx`, +`robots.ts`. If a route directory is still there, move it. `git status --short` +is the authoritative check: untracked files do not show in `git diff --stat`. + +- [ ] **Step 3: Update the four test imports** + +In `__tests__/app/metadata.test.ts`, change lines 1-4 to: + +```ts +import { metadata as homeMetadata } from '@/app/(frontend)/page'; +import { metadata as rankingsMetadata } from '@/app/(frontend)/rankings/page'; +import { metadata as admissionsMetadata } from '@/app/(frontend)/admissions/page'; +import { generateMetadata as compareMetadata } from '@/app/(frontend)/compare/page'; +``` + +In `__tests__/app/placeMetadata.test.ts`, line 1: + +```ts +import { generateMetadata as placeMeta } from '@/app/(frontend)/schools/[place]/page'; +``` + +In `__tests__/api/proxyDenylist.test.ts`, line 11: + +```ts +import { GET } from '@/app/(frontend)/api/[...path]/route'; +``` + +- [ ] **Step 4: Find any other references to the old paths** + +Two searches, because import specifiers are not the only way a file names a +path. Quote the `--include` globs or zsh expands them. + +```bash +cd /Users/tudor/projects/school_compare +# 1. Import specifiers +grep -rn "app/layout\|app/page\|@/app/" \ + --include="*.ts" --include="*.tsx" --include="*.js" --include="*.mjs" \ + nextjs-app --exclude-dir=node_modules --exclude-dir=.next | grep -v "app/(frontend)" + +# 2. Filesystem paths — readFileSync/path.join targets, which search 1 misses. +# __tests__/components/darkThemeSafety.test.ts reads app/globals.css this way +# and fails to *run* when the path is stale, so it shows as a failed suite +# rather than a failed assertion. +grep -rn "'app'" nextjs-app/__tests__ --include="*.ts" --include="*.tsx" \ + | grep -v "(frontend)" +``` + +Expected: no output from either. Fix anything that appears — note that +`renderSchoolDetail.tsx` names `app/layout.tsx` in a comment, which should be +updated for accuracy even though nothing breaks. + +- [ ] **Step 5: Verify tests, types and build** + +```bash +cd nextjs-app && npm test && npm run typecheck && npm run build +``` + +Expected: the same passing count as Step 1, and a successful build. In the +build's route table, confirm the routes are still listed as `/`, `/rankings`, +`/compare`, `/admissions`, `/school/[slug]` — **not** `/(frontend)/...`. If the +group name appears in a URL, the directory was named wrongly (it must include +the parentheses). + +- [ ] **Step 6: Commit** + +```bash +git add -A nextjs-app +git commit -m "refactor(app): move site routes into a (frontend) route group + +Payload's admin panel ships its own root layout rendering html/body. +Next allows multiple root layouts only when no app/layout.tsx exists, so +the site's routes move into their own group. Route groups are invisible +to routing: every public URL is unchanged." +``` + +--- + +### Task 3: Install Payload and bring up `/admin` + +**Files:** +- Modify: `nextjs-app/package.json` +- Modify: `nextjs-app/tsconfig.json` +- Modify: `nextjs-app/next.config.mjs` +- Create: `nextjs-app/payload.config.ts` +- Create: `nextjs-app/collections/Users.ts` +- Create: `nextjs-app/lib/payload.ts` +- Create: `nextjs-app/app/(payload)/**` (generated) +- Create: `nextjs-app/__tests__/payload/config.test.ts` + +**Interfaces:** +- Consumes: Task 2's absent root layout. +- Produces: `payload.config.ts` default-exporting the built config; + `getCachedPayload(): Promise` from `lib/payload.ts`, used by Tasks 7 + and 8 to query content. + +- [ ] **Step 1: Install the packages** + +```bash +cd nextjs-app +npm install payload @payloadcms/db-postgres @payloadcms/next @payloadcms/richtext-lexical graphql +npm install sharp +npm uninstall --save-dev sharp +``` + +`sharp` moves from `devDependencies` to `dependencies`: Payload needs it at +runtime to generate `imageSizes`, and the Docker runner stage installs +production dependencies only. + +- [ ] **Step 2: Add the `@payload-config` path alias** + +In `nextjs-app/tsconfig.json`, extend `compilerOptions.paths` (currently +`{"@/*": ["./*"]}`) to: + +```json +"paths": { + "@/*": ["./*"], + "@payload-config": ["./payload.config.ts"] +} +``` + +- [ ] **Step 3: Write the failing config test** + +Create `nextjs-app/__tests__/payload/config.test.ts`: + +```ts +/** + * @jest-environment node + */ +import config from '@/payload.config'; + +describe('payload config', () => { + it('serves its API from /cms-api, not /api', async () => { + // app/(frontend)/api/[...path]/route.ts is a catch-all proxying /api/* + // to FastAPI. Payload's default /api would be swallowed by it, and the + // failure is silent — admin calls would be forwarded to the backend. + const resolved = await config; + expect(resolved.routes.api).toBe('/cms-api'); + }); + + it('serves the admin panel from /admin', async () => { + const resolved = await config; + expect(resolved.routes.admin).toBe('/admin'); + }); + + it('authenticates against the users collection', async () => { + const resolved = await config; + expect(resolved.admin.user).toBe('users'); + }); +}); +``` + +- [ ] **Step 4: Run it to verify it fails** + +```bash +cd nextjs-app && npm test -- __tests__/payload/config.test.ts +``` + +Expected: FAIL — cannot resolve `@/payload.config`. + +- [ ] **Step 5: Write the Users collection** + +Create `nextjs-app/collections/Users.ts`: + +```ts +import type { CollectionConfig } from 'payload'; + +/** + * The site's only authenticated surface. There is one account and no + * registration: `create` is closed to everyone, so the first user is seeded + * with `payload create-first-user` and no one can add another through the API. + */ +export const Users: CollectionConfig = { + slug: 'users', + auth: { + // Slows credential stuffing against a panel that is on the public + // internet. Five attempts, then a ten-minute lock. + maxLoginAttempts: 5, + lockTime: 10 * 60 * 1000, + }, + access: { + create: () => false, + read: ({ req }) => Boolean(req.user), + update: ({ req }) => Boolean(req.user), + delete: () => false, + }, + admin: { useAsTitle: 'email' }, + fields: [ + { + name: 'displayName', + type: 'text', + required: true, + // Rendered as the byline on every post. First name only — see the + // author identity constraint at the top of this plan. + defaultValue: 'Tudor', + }, + ], +}; +``` + +- [ ] **Step 6: Write the Payload config** + +Create `nextjs-app/payload.config.ts`: + +```ts +import path from 'path'; +import { fileURLToPath } from 'url'; +import { buildConfig } from 'payload'; +import { postgresAdapter } from '@payloadcms/db-postgres'; +import { lexicalEditor } from '@payloadcms/richtext-lexical'; +import sharp from 'sharp'; +import { Users } from '@/collections/Users'; + +const filename = fileURLToPath(import.meta.url); +const dirname = path.dirname(filename); + +export default buildConfig({ + admin: { user: Users.slug }, + // /api belongs to the FastAPI proxy. See the config test for why this + // must never move back. + routes: { api: '/cms-api', admin: '/admin' }, + collections: [Users], + editor: lexicalEditor(), + secret: process.env.PAYLOAD_SECRET || '', + typescript: { outputFile: path.resolve(dirname, 'payload-types.ts') }, + db: postgresAdapter({ + pool: { connectionString: process.env.DATABASE_URL }, + // Its own schema, so no pipeline operation on `public` can reach blog + // content. scripts/migrate_csv_to_db.py --drop lives in that blast radius. + schemaName: 'payload', + }), + sharp, +}); +``` + +- [ ] **Step 7: Generate the Payload route files** + +```bash +cd nextjs-app && npx create-payload-app@latest --no-deps +``` + +Choose the options for adding to an existing Next.js app. This generates +`app/(payload)/` — the admin panel routes, the `/cms-api` route handlers, and +Payload's own root layout. **Do not hand-write these files**; they are +version-coupled to the installed Payload release and are regenerated on upgrade. + +Then confirm the generated API directory matches the remapped route: + +```bash +cd nextjs-app && ls "app/(payload)" +``` + +If the generated directory is `app/(payload)/api`, rename it to match +`routes.api`: + +```bash +cd nextjs-app && git mv "app/(payload)/api" "app/(payload)/cms-api" +``` + +If the generator overwrote `payload.config.ts`, restore the version from +Step 6 — the `routes` and `schemaName` keys are the whole point and the +generator does not write them. + +- [ ] **Step 8: Wrap the Next config** + +In `nextjs-app/next.config.mjs`, add at the top: + +```js +import { withPayload } from '@payloadcms/next/withPayload'; +``` + +and change the final line from `export default nextConfig;` to: + +```js +export default withPayload(nextConfig); +``` + +- [ ] **Step 9: Add the cached Payload accessor** + +Create `nextjs-app/lib/payload.ts`: + +```ts +import { getPayload } from 'payload'; +import config from '@payload-config'; +import type { Payload } from 'payload'; + +/** + * One Payload instance per process. getPayload() is itself memoised by + * Payload, but routing every caller through here keeps the config import in + * a single place and gives page code one name to mock in tests. + */ +export function getCachedPayload(): Promise { + return getPayload({ config }); +} +``` + +- [ ] **Step 10: Run the config test** + +```bash +cd nextjs-app && npm test -- __tests__/payload/config.test.ts +``` + +Expected: PASS, 3 tests. + +- [ ] **Step 11: Verify types and build** + +```bash +cd nextjs-app && npm test && npm run typecheck && npm run build +``` + +Expected: all green. The build must succeed **without** a database — nothing +added so far queries Payload at build time. If the build tries to connect, +something imported `getCachedPayload()` at module scope; move it inside the +request handler. + +- [ ] **Step 12: Commit** + +```bash +git add -A nextjs-app +git commit -m "feat(cms): install Payload and serve the admin panel + +Payload runs inside the Next app against the existing Postgres, in its +own 'payload' schema so no pipeline operation on public can reach blog +content. Its API is remapped to /cms-api because /api is the FastAPI +proxy's catch-all." +``` + +--- + +### Task 4: Harden the admin surface + +`/admin` is the first authenticated surface on this site. It must never be +indexed and must not be reachable through search results. + +**Files:** +- Modify: `nextjs-app/app/robots.ts` +- Modify: `nextjs-app/next.config.mjs` +- Modify: `nextjs-app/__tests__/app/nextConfig.test.ts` +- Create: `nextjs-app/__tests__/app/robots.test.ts` + +**Interfaces:** +- Consumes: Task 3's `/admin` and `/cms-api` routes. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Write the failing tests** + +Create `nextjs-app/__tests__/app/robots.test.ts`: + +```ts +import robots from '@/app/robots'; + +describe('robots.txt', () => { + it('disallows the admin panel and the CMS API', () => { + const rules = robots().rules; + const rule = Array.isArray(rules) ? rules[0] : rules; + expect(rule.disallow).toEqual( + expect.arrayContaining(['/api/', '/_next/', '/admin/', '/cms-api/']), + ); + }); +}); +``` + +Append to `nextjs-app/__tests__/app/nextConfig.test.ts`: + +```ts +describe('admin surface', () => { + it('serves noindex on the admin panel and the CMS API', async () => { + const headers = await nextConfig.headers(); + for (const source of ['/admin/:path*', '/cms-api/:path*']) { + const rule = headers.find((entry) => entry.source === source); + expect(rule).toBeDefined(); + expect(rule.headers).toContainEqual({ + key: 'X-Robots-Tag', + value: 'noindex, nofollow', + }); + } + }); +}); +``` + +- [ ] **Step 2: Run them to verify they fail** + +```bash +cd nextjs-app && npm test -- __tests__/app/robots.test.ts __tests__/app/nextConfig.test.ts +``` + +Expected: FAIL — the disallow list lacks the new entries, and neither header +rule exists. + +- [ ] **Step 3: Add the robots disallow entries** + +In `nextjs-app/app/robots.ts`, change the `disallow` array to: + +```ts +disallow: ['/api/', '/_next/', '/admin/', '/cms-api/'], +``` + +- [ ] **Step 4: Add the noindex headers** + +In `nextjs-app/next.config.mjs`, add these two entries to the array returned by +`headers()`, immediately after the staging-host rule: + +```js +{ + /* + * The admin panel and the CMS API must never be indexed. robots.txt + * disallows them too, but a Disallow only blocks crawling — a URL found + * from an external link can still be indexed without ever being fetched. + * This header is what actually keeps them out. + */ + source: '/admin/:path*', + headers: [{ key: 'X-Robots-Tag', value: 'noindex, nofollow' }], +}, +{ + source: '/cms-api/:path*', + headers: [{ key: 'X-Robots-Tag', value: 'noindex, nofollow' }], +}, +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +```bash +cd nextjs-app && npm test -- __tests__/app/robots.test.ts __tests__/app/nextConfig.test.ts +``` + +Expected: PASS. + +- [ ] **Step 6: Confirm the CSP does not block the admin panel** + +The existing global CSP sets `frame-ancestors 'self' https://analytics.schoolcompare.co.uk`. +`frame-ancestors` restricts who may embed this site; it does not restrict +scripts, styles or fetches, so it cannot break the admin panel. Confirm no +other CSP directive was added, then move on: + +```bash +cd nextjs-app && grep -n "Content-Security-Policy" -A2 next.config.mjs +``` + +Expected: only the `frame-ancestors` directive. + +- [ ] **Step 7: Full verification and commit** + +```bash +cd nextjs-app && npm test && npm run typecheck && npm run build +git add -A nextjs-app +git commit -m "feat(cms): keep the admin panel out of the index + +X-Robots-Tag rather than robots.txt alone: a Disallow blocks crawling, +not indexing, and a disallowed URL found from an external link can be +indexed without ever being fetched." +``` + +--- + +### Task 5: Make the stack deployable + +**Files:** +- Modify: `nextjs-app/Dockerfile` +- Modify: `docker-compose.portainer.yml` +- Modify: `docker-compose.portainer.staging.yml` +- Modify: `nextjs-app/payload.config.ts` +- Create: `nextjs-app/migrations/` (generated) + +**Interfaces:** +- Consumes: Task 3's `payload.config.ts`. +- Produces: a container that runs migrations on boot and can write uploads to + `/app/media`. + +- [ ] **Step 1: Generate the initial migration** + +With a local Postgres reachable and `DATABASE_URL` / `PAYLOAD_SECRET` exported: + +```bash +cd nextjs-app && npx payload migrate:create initial +``` + +This writes `migrations/_initial.ts` and `migrations/index.ts`. +Both are committed — schema changes travel through the same PR and staging gate +as code. + +- [ ] **Step 2: Run migrations on boot in production** + +In `nextjs-app/payload.config.ts`, add the import and the adapter key: + +```ts +import { migrations } from '@/migrations'; +``` + +```ts + db: postgresAdapter({ + pool: { connectionString: process.env.DATABASE_URL }, + schemaName: 'payload', + // Runs pending migrations during server init. Preferred over a one-shot + // init container (the airflow-init pattern) because this is a single + // long-running process with no ordering problem to solve. + prodMigrations: migrations, + }), +``` + +- [ ] **Step 3: Create the media directory in the image** + +In `nextjs-app/Dockerfile`, in the **runner** stage, immediately after the +`COPY --from=builder /app/assets ./assets` line and **before** +`RUN chown -R nextjs:nodejs /app`, add: + +```dockerfile +# Payload writes uploads here, and the compose file mounts a named volume +# over it. The directory must exist and be owned by the runtime user BEFORE +# the mount: Docker seeds a fresh named volume from the image path, so a +# missing or root-owned directory here makes every upload fail with EACCES — +# at runtime, long after the build passed. +RUN mkdir -p /app/media +``` + +`RUN chown -R nextjs:nodejs /app` already follows and covers it. + +- [ ] **Step 4: Wire the frontend service in production compose** + +In `docker-compose.portainer.yml`, under `services.frontend`, add to +`environment`: + +```yaml + - DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@sc_database:5432/${DB_DATABASE_NAME} + - PAYLOAD_SECRET=${PAYLOAD_SECRET:?set PAYLOAD_SECRET in the Portainer stack environment} +``` + +Add to the service: + +```yaml + volumes: + - payload_media:/app/media +``` + +Add `sc_database` to its `depends_on`: + +```yaml + depends_on: + backend: + condition: service_healthy + sc_database: + condition: service_healthy +``` + +Add to the top-level `volumes:` block: + +```yaml + payload_media: +``` + +Add to the header comment block, alongside the other documented variables: + +``` +# PAYLOAD_SECRET — Payload CMS encryption secret. REQUIRED: long and +# random. Changing it invalidates all admin sessions. +# Staging MUST use a different value from production. +``` + +The `:?` form is deliberate and matches `AIRFLOW_ADMIN_PASSWORD` above it: the +container refuses to start rather than booting with an empty secret. + +- [ ] **Step 5: Apply the same changes to staging** + +Repeat Step 4 in `docker-compose.portainer.staging.yml`, adapting service and +host names to that file's conventions. Staging gets its **own** +`PAYLOAD_SECRET` and its own admin credentials — never production's. + +- [ ] **Step 6: Verify the build** + +```bash +cd nextjs-app && npm run typecheck && npm run build +docker build -t sc-frontend-test nextjs-app +``` + +Expected: both succeed. The Docker build proves the `mkdir`/`chown` ordering +and that `sharp` resolves as a production dependency. + +- [ ] **Step 7: Commit** + +```bash +git add -A nextjs-app docker-compose.portainer.yml docker-compose.portainer.staging.yml +git commit -m "build(cms): make the Payload-enabled image deployable + +Migrations run on server init via prodMigrations. Uploads go to a named +volume at /app/media, created and chowned in the image before the mount +so Docker does not seed it root-owned and fail every upload at runtime." +``` + +- [ ] **Step 8: Note the operational follow-ups for the human** + +Record these in the PR description — they are not code: +- Set `PAYLOAD_SECRET` in both Portainer stacks (different values). +- Add the `payload_media` volume to the backup routine. Post images are not + reproducible from the pipeline. +- After first deploy, run `npx payload create-first-user` against the + container to seed the single admin account. +- Confirm `scripts/migrate_csv_to_db.py --drop` is schema-scoped and cannot + reach the `payload` schema. + +--- + +### Task 6: The About page + +**Files:** +- Create: `nextjs-app/app/(frontend)/about/page.tsx` +- Create: `nextjs-app/app/(frontend)/about/About.module.css` +- Create: `nextjs-app/lib/jsonld.ts` +- Create: `nextjs-app/__tests__/app/aboutMetadata.test.ts` +- Add: `nextjs-app/public/brand/tudor.jpg` (supplied by the human) +- Modify: `nextjs-app/components/Footer.tsx` +- Modify: `e2e/tests/journeys.spec.ts` + +**Interfaces:** +- Consumes: `absoluteUrl` from `lib/site.ts`. +- Produces: `personJsonLd()` and `organizationJsonLd()` from `lib/jsonld.ts`, + reused by Task 8's `BlogPosting.author`. + +- [ ] **Step 1: Write the failing tests** + +Create `nextjs-app/__tests__/app/aboutMetadata.test.ts`: + +```ts +import { metadata } from '@/app/(frontend)/about/page'; +import { personJsonLd, organizationJsonLd } from '@/lib/jsonld'; + +describe('/about metadata', () => { + it('canonicalises to the bare path', () => { + expect(metadata.alternates?.canonical) + .toBe('https://www.schoolcompare.co.uk/about'); + }); +}); + +describe('author structured data', () => { + it('describes a Person with a first name and a photo', () => { + const person = personJsonLd(); + expect(person['@type']).toBe('Person'); + expect(person.name).toBe('Tudor'); + expect(person.image).toBe('https://www.schoolcompare.co.uk/brand/tudor.jpg'); + expect(person.url).toBe('https://www.schoolcompare.co.uk/about'); + }); + + it('never publishes a surname', () => { + // Author identity constraint: first name only, no employer. + expect(JSON.stringify(personJsonLd())).not.toMatch(/familyName|Sitaru/i); + }); + + it('describes the site as an Organization the Person authors for', () => { + const org = organizationJsonLd(); + expect(org['@type']).toBe('Organization'); + expect(org.name).toBe('schoolcompare'); + expect(org.url).toBe('https://www.schoolcompare.co.uk'); + }); +}); +``` + +- [ ] **Step 2: Run to verify failure** + +```bash +cd nextjs-app && npm test -- __tests__/app/aboutMetadata.test.ts +``` + +Expected: FAIL — neither module exists. + +- [ ] **Step 3: Write the JSON-LD builders** + +Create `nextjs-app/lib/jsonld.ts`: + +```ts +import { SITE_URL, absoluteUrl } from '@/lib/site'; + +/** + * The site's author entity. First name only, by choice: see the About page. + * Everything that needs an author — the About page, every post byline — + * references this one shape so the entity stays consistent for search. + */ +export function personJsonLd() { + return { + '@type': 'Person', + '@id': `${SITE_URL}/about#tudor`, + name: 'Tudor', + url: absoluteUrl('/about'), + image: absoluteUrl('/brand/tudor.jpg'), + description: + 'Parent in south-west London who built schoolcompare while looking for a primary school.', + } as const; +} + +export function organizationJsonLd() { + return { + '@type': 'Organization', + '@id': `${SITE_URL}#organization`, + name: 'schoolcompare', + url: SITE_URL, + logo: absoluteUrl('/icon-512.png'), + } as const; +} +``` + +- [ ] **Step 4: Write the About page** + +Create `nextjs-app/app/(frontend)/about/page.tsx`. The copy below is the +deliverable — write it as given. It follows the voice rules and the "not an +education expert" constraint. + +```tsx +import type { Metadata } from 'next'; +import Image from 'next/image'; +import { absoluteUrl } from '@/lib/site'; +import { personJsonLd, organizationJsonLd } from '@/lib/jsonld'; +import styles from './About.module.css'; + +export const metadata: Metadata = { + title: 'About', + description: + 'Who builds schoolcompare, why it exists, and where its numbers come from.', + alternates: { canonical: absoluteUrl('/about') }, +}; + +export default function AboutPage() { + const jsonLd = { + '@context': 'https://schema.org', + '@graph': [personJsonLd(), organizationJsonLd()], + }; + + return ( +
+