From 0c901cd0d1fe404334b034a16b49b9eafef2d22b Mon Sep 17 00:00:00 2001 From: Tudor Date: Tue, 15 Sep 2026 10:17:50 +0100 Subject: [PATCH] feat(ci): gate promotion on the image set that actually passed E2E MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staging health polling asked only whether something answered HTTP 200 at the base URL. It could not tell the new deployment from the old one, so journeys could pass against the previous release, and concurrent merges could move the staging tags underneath a run in flight. Each staging run now mints a build ID and stamps all three images with the commit and that ID, as labels and — for frontend and backend — as a build-time JSON file that environment overrides cannot rewrite. /release.json reports both identities uncached, and scripts/ci/release.py polls for the expected pair before and after the journeys. Only then are the captured build digests tagged verified-. Promotion resolves those verified tags to immutable digests, revalidates their labels, and refuses a mixed or incomplete set before any :prod tag moves. The whole staging workflow shares one concurrency group with cancellation disabled, so releases serialise. The scripts are stdlib-only and unit-tested against mocked registry and HTTP behaviour; PR checks now run the pipeline and CI suites too. The runbook records what this cannot prove locally, and that the first rollout needs a commit built by this workflow. Co-Authored-By: Claude Opus 5 --- .gitea/workflows/deploy.yml | 90 +++++++++--- .gitea/workflows/pr-checks.yml | 4 +- .gitea/workflows/promote.yml | 32 +---- Dockerfile | 6 + docs/ARCHITECTURE.md | 33 +++-- docs/DEPLOY.md | 48 ++++++- docs/DEVELOPMENT.md | 4 +- e2e/tests/reliability.spec.ts | 41 ++++++ nextjs-app/Dockerfile | 10 ++ .../__tests__/app/releaseIdentity.test.ts | 30 ++++ .../app/(frontend)/release.json/route.ts | 17 +++ pipeline/Dockerfile | 5 + scripts/ci/release.py | 136 ++++++++++++++++++ scripts/ci/tests/test_release.py | 65 +++++++++ scripts/ci/tests/test_workflows.py | 34 +++++ 15 files changed, 494 insertions(+), 61 deletions(-) create mode 100644 e2e/tests/reliability.spec.ts create mode 100644 nextjs-app/__tests__/app/releaseIdentity.test.ts create mode 100644 nextjs-app/app/(frontend)/release.json/route.ts create mode 100644 scripts/ci/release.py create mode 100644 scripts/ci/tests/test_release.py create mode 100644 scripts/ci/tests/test_workflows.py diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 31bb6ea..f24cb71 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -5,6 +5,11 @@ on: branches: - main +# Serialise the entire build/deploy/test cycle: no other run can move staging tags. +concurrency: + group: staging-release + cancel-in-progress: false + env: REGISTRY: privaterepo.sitaru.org BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend @@ -12,7 +17,18 @@ env: PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline jobs: + prepare: + runs-on: ubuntu-latest + outputs: + build_id: ${{ steps.identity.outputs.build_id }} + steps: + - id: identity + run: python3 -c 'import uuid; print("build_id=" + uuid.uuid4().hex)' >> "$GITHUB_OUTPUT" + build-backend: + needs: [prepare] + outputs: + digest: ${{ steps.build.outputs.digest }} name: Build Backend (FastAPI) runs-on: ubuntu-latest steps: @@ -46,17 +62,24 @@ jobs: type=raw,value=staging - name: Build and push Backend Docker image + id: build uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true + build-args: | + BUILD_SHA=${{ gitea.sha }} + BUILD_ID=${{ needs.prepare.outputs.build_id }} tags: ${{ steps.meta-backend.outputs.tags }} labels: ${{ steps.meta-backend.outputs.labels }} cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:buildcache,mode=max build-frontend: + needs: [prepare] + outputs: + digest: ${{ steps.build.outputs.digest }} name: Build Frontend (Next.js) runs-on: ubuntu-latest steps: @@ -90,18 +113,23 @@ jobs: type=raw,value=staging - name: Build and push Frontend Docker image + id: build uses: docker/build-push-action@v5 with: context: ./nextjs-app file: ./nextjs-app/Dockerfile push: true + build-args: | + BUILD_SHA=${{ gitea.sha }} + BUILD_ID=${{ needs.prepare.outputs.build_id }} tags: ${{ steps.meta-frontend.outputs.tags }} labels: ${{ steps.meta-frontend.outputs.labels }} - build-args: | - FASTAPI_URL=http://backend:80/api # Cache disabled due to registry size limits build-pipeline: + needs: [prepare] + outputs: + digest: ${{ steps.build.outputs.digest }} name: Build Pipeline (Meltano + dbt + Airflow) runs-on: ubuntu-latest steps: @@ -135,11 +163,15 @@ jobs: type=raw,value=staging - name: Build and push Pipeline Docker image + id: build uses: docker/build-push-action@v5 with: context: ./pipeline file: ./pipeline/Dockerfile push: true + build-args: | + BUILD_SHA=${{ gitea.sha }} + BUILD_ID=${{ needs.prepare.outputs.build_id }} tags: ${{ steps.meta-pipeline.outputs.tags }} labels: ${{ steps.meta-pipeline.outputs.labels }} cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PIPELINE_IMAGE_NAME }}:buildcache @@ -148,30 +180,23 @@ jobs: deploy-staging: name: Deploy to Staging runs-on: ubuntu-latest - needs: [build-backend, build-frontend, build-pipeline] + needs: [prepare, build-backend, build-frontend, build-pipeline] steps: - name: Trigger staging stack update run: curl -fsSk -X POST "${{ secrets.PORTAINER_STAGING_WEBHOOK }}" - - name: Wait for staging to become healthy - run: | - echo "Polling ${STAGING_BASE_URL} for up to 5 minutes..." - for i in $(seq 1 60); do - if curl -fsS -o /dev/null --max-time 10 "${STAGING_BASE_URL}/"; then - echo "Staging is up (attempt $i)" - exit 0 - fi - sleep 5 - done - echo "Staging did not become healthy in time" >&2 - exit 1 + - uses: actions/checkout@v4 + - name: Verify deployed release identity + run: python3 scripts/ci/release.py wait env: - STAGING_BASE_URL: ${{ secrets.STAGING_BASE_URL }} + BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} e2e-staging: name: E2E Journeys against Staging runs-on: ubuntu-latest - needs: [deploy-staging] + needs: [prepare, deploy-staging, build-backend, build-frontend, build-pipeline] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -187,11 +212,42 @@ jobs: npm ci npx playwright install --with-deps chromium + - name: Verify release before journeys + run: python3 scripts/ci/release.py wait --timeout 10 + env: + BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + - name: Run E2E journeys working-directory: e2e run: npx playwright test env: BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + + - name: Verify release after journeys + run: python3 scripts/ci/release.py wait --timeout 10 + env: + BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ gitea.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Mark tested image digests as verified + run: python3 scripts/ci/release.py verify + env: + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + BACKEND_DIGEST: ${{ needs.build-backend.outputs.digest }} + FRONTEND_DIGEST: ${{ needs.build-frontend.outputs.digest }} + PIPELINE_DIGEST: ${{ needs.build-pipeline.outputs.digest }} # Production deployment is a second, manual approval: see promote.yml # ("Promote to Production (manual)") and docs/DEPLOY.md. diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index b7d3a11..09a843d 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -68,13 +68,13 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install -r requirements.txt pytest "httpx<0.28" + run: pip install -r requirements.txt pytest "httpx<0.28" pyyaml - name: Import smoke test run: python -c "from backend.app import app; print('backend imports OK')" - name: Backend unit tests - run: python -m pytest backend/tests -q + run: python -m pytest backend/tests pipeline/tests scripts/ci/tests -q build-backend: name: Build Backend (no push) diff --git a/.gitea/workflows/promote.yml b/.gitea/workflows/promote.yml index 777530c..35652bb 100644 --- a/.gitea/workflows/promote.yml +++ b/.gitea/workflows/promote.yml @@ -97,33 +97,15 @@ jobs: username: ${{ gitea.actor }} password: ${{ secrets.REGISTRY_TOKEN }} - - name: Retag approved images as prod (keeping rollback pointer) - run: | - SHORT_SHA="${{ steps.resolve.outputs.short }}" - for IMAGE in \ - "${REGISTRY}/${BACKEND_IMAGE_NAME}" \ - "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \ - "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do - # Keep a rollback pointer before moving :prod - docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true - docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}" - echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod" - done + - name: Resolve verified digests and promote the complete image set + run: python3 scripts/ci/release.py promote --output release.json + env: + EXPECTED_SHA: ${{ steps.resolve.outputs.full }} - name: Trigger production stack update run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}" - - name: Wait for production to become healthy - run: | - echo "Polling ${PROD_BASE_URL} for up to 5 minutes..." - for i in $(seq 1 60); do - if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then - echo "Production is up (attempt $i)" - exit 0 - fi - sleep 5 - done - echo "Production did not become healthy in time" >&2 - exit 1 + - name: Verify production release identity + run: python3 scripts/ci/release.py wait --release release.json env: - PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }} + BASE_URL: ${{ secrets.PROD_BASE_URL }} diff --git a/Dockerfile b/Dockerfile index 6b9011f..c048914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,12 @@ RUN pip install --no-cache-dir -r requirements.txt COPY backend/ ./backend/ COPY scripts/ ./scripts/ +ARG BUILD_SHA=development +ARG BUILD_ID=development +LABEL io.schoolcompare.build-id=$BUILD_ID +LABEL io.schoolcompare.commit=$BUILD_SHA +RUN python -c 'import json,sys; open("backend/build-info.json", "w").write(json.dumps({"sha":sys.argv[1],"build_id":sys.argv[2]}))' "$BUILD_SHA" "$BUILD_ID" + # Expose the application port EXPOSE 80 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4ea42cd..a00995e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -80,9 +80,11 @@ API types. `payload-types.ts` and the Payload import map are generated artifacts 1. Airflow DAGs extract and validate source data, then run selected dbt builds. 2. Relevant DAGs rebuild Typesense and swap the `schools` alias. -3. They call `POST /api/admin/reload` with `X-API-Key` to refresh school DataFrames. -4. A separate weekly sitemap DAG calls `POST /api/admin/regenerate-sitemap`, - rebuilding places and sitemaps. +3. They call `POST /api/admin/reload` with `X-API-Key`. It builds and validates + replacement DataFrames, places, reverse membership and sitemaps off the request + loop, then publishes them together. Failure returns 503 and preserves live data. +4. A separate weekly sitemap DAG can regenerate the derived publication from the + current DataFrame without clearing the live registry first. GIAS is scheduled daily, Ofsted monthly, and annual datasets are manually triggered. The DAG definitions are authoritative for selectors and dependencies. @@ -92,16 +94,25 @@ backend HTTP Cache-Control/ETags, Next.js fetch/page revalidation, and browser o shared HTTP caches where configured. Place fetches request a one-week revalidation interval. HTTP ETags are computed after route execution, not before database work. -Known limitations: reload clears the old DataFrames before verifying replacement -data; places/sitemaps refresh separately; Next.js caches are not explicitly purged -by the pipeline; Typesense import results are not validated before alias publication. -Do not describe this sequence as an atomic dataset release. These are follow-up -reliability tasks, not changes implemented by the documentation cleanup. +Typesense publication validates every import response and the final document +count before switching aliases. A session-scoped PostgreSQL advisory lock +serialises index reads/publication across DAGs. The previous collection remains +available for rollback; old unaliased collections are pruned after success. +Failed drafts are retained until a later successful cleanup, because an uncertain +alias-update response must never cause deletion of a potentially live index. + +The backend snapshot swap is process-local and assumes the current single-worker +deployment. It is not an atomic transaction spanning PostgreSQL marts, Typesense +and Next.js caches. Next.js caches are not explicitly purged by the pipeline. +School search retrieves every Typesense candidate before applying API filters; +only a dependency failure invokes substring fallback, not a valid empty match set. ## Deployment references See [DEPLOY.md](DEPLOY.md). PR checks include frontend typechecking/tests, backend unit tests, image builds and AI review. Staging journeys run after merging. -Production promotion retags a selected commit's images. Current health polling -checks HTTP success, not the deployed commit identity; overlapping staging runs -remain a release-verification concern. +Staging runs are serialised across builds, deployment and E2E. Build-stamped +frontend/backend identities are checked before and after journeys. Only then are +the captured image digests marked verified. Promotion resolves and validates the +complete verified image set before retagging production. See the runbook for +first-rollout requirements and remaining integration checks. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 2cff1b8..63e85e0 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -19,8 +19,9 @@ PR checks (.gitea/workflows/pr-checks.yml) ▼ Stage pipeline (.gitea/workflows/deploy.yml) — automatic 1. build & push images → tags sha-, staging - 2. staging Portainer webhook → wait for staging health + 2. staging Portainer webhook → verify frontend/backend SHA + build ID 3. Playwright E2E journeys against staging ← gate before human testing + 4. verify identity again; tag tested digests verified- ▼ Manual testing on staging (stx.schoolcompare.co.uk) │ Actions → "Promote to Production (manual)" ← approval #2 @@ -28,14 +29,15 @@ Manual testing on staging (stx.schoolcompare.co.uk) Promote pipeline (.gitea/workflows/promote.yml) — manual dispatch 1. resolve target sha (input, or latest main if empty) 2. REFUSE unless that commit's "E2E Journeys against Staging" status is green - 3. retag sha- → :prod (same bytes — build once, promote the image) + 3. resolve verified- digests, validate labels, retag digests → :prod previous :prod saved as :prod-previous - 4. prod Portainer webhook → wait for prod health + 4. prod Portainer webhook → verify expected SHA + build ID ``` Key principle: **build once, promote the exact image**. Production pins `:prod`, which only moves when a human runs the promote workflow — and the workflow -only accepts commits that passed the staging E2E gate. Nothing tags `:latest` +only accepts commits that passed the staging E2E gate and have a complete verified +image set. Nothing tags `:latest` anymore. ## Branch & PR workflow @@ -256,3 +258,41 @@ how long any feature is exposed to this. If `UNLEASH_URL` is unset, every flag is `False` and no connection is attempted. That is the correct behaviour for local development and CI, and it means the test suites need no flag server. + +## Release identity and the P1 reliability gate + +Every staging run creates a random build ID before building its three images. +Each image carries the commit and build ID as labels. Frontend/backend images +also contain a build-time JSON file; environment overrides cannot rewrite it. +`/release.json` returns both identities with `Cache-Control: no-store`. It fails +with 503 when either identity cannot be read. FastAPI's internal endpoint is +`/api/release`. + +The entire staging workflow shares one concurrency group, with cancellation +disabled. This needs Gitea 1.26 or newer, where workflow concurrency is supported +([release notes](https://blog.gitea.com/release-of-1.26.0/)); the configured server +reported 1.27.3 during this change. Do not run the workflow on an older server +that ignores the concurrency key. Manual deployments outside this workflow must +also avoid changing staging during journeys. + +The gate checks both identities before and after Playwright. It then validates +labels on the captured build output digests and tags them `verified-`. +The manual promotion script resolves all three verified tags to immutable digests +and confirms one matching commit/build ID before moving any `:prod` tag. It polls +production for that same identity using a locally saved release manifest. +A registry error can still interrupt the three tag writes; the Portainer webhook +only runs after successful promotion, and rerunning promotion resolves the full +verified set again. There is no cross-registry atomic tag transaction. + +**First rollout:** old green commits without verified tags/build identities are +not promotable through this gate. Build and test a commit containing the new +workflow first. The release route must be reachable through the configured +`STAGING_BASE_URL`/`PROD_BASE_URL`; it deliberately avoids the public staging +`/api` proxy limitation. No new deployment secret is required. + +`scripts/ci/release.py` implements identity polling and digest verification. +Its mocked tests run in PR checks alongside backend and index-publication tests. +The new Playwright journeys also check deployed identity and stale pagination. +Local unit checks do not validate registry credentials, Portainer behaviour, +proxy routing or a deployed image; those require the staging run. Production +promotion remains a separate human action. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index f1ebb0c..613cc85 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -42,8 +42,8 @@ From the repository root, using an available Python 3.11 or 3.12 interpreter: ```sh python3.11 -m venv /tmp/schoolcompare-backend-venv -/tmp/schoolcompare-backend-venv/bin/python -m pip install -r requirements.txt pytest 'httpx<0.28' -/tmp/schoolcompare-backend-venv/bin/python -m pytest backend/tests -q +/tmp/schoolcompare-backend-venv/bin/python -m pip install -r requirements.txt pytest 'httpx<0.28' pyyaml +/tmp/schoolcompare-backend-venv/bin/python -m pytest backend/tests pipeline/tests scripts/ci/tests -q ``` Substitute `python3.12` if matching PR CI. The test dependencies above match the diff --git a/e2e/tests/reliability.spec.ts b/e2e/tests/reliability.spec.ts new file mode 100644 index 0000000..31a2d4a --- /dev/null +++ b/e2e/tests/reliability.spec.ts @@ -0,0 +1,41 @@ +import { test, expect, Route } from '@playwright/test'; + +test('the deployed frontend and backend report the tested build', async ({ request }) => { + const response = await request.get('/release.json'); + expect(response.ok()).toBeTruthy(); + expect(response.headers()['cache-control']).toContain('no-store'); + const identity = await response.json(); + expect(identity.frontend).toEqual(identity.backend); + expect(identity.frontend.sha).toMatch(/^[a-f0-9]{40}$/); + expect(identity.frontend.build_id).toMatch(/^[a-f0-9]{32}$/); + if (process.env.EXPECTED_SHA) expect(identity.frontend.sha).toBe(process.env.EXPECTED_SHA); + if (process.env.EXPECTED_BUILD_ID) expect(identity.frontend.build_id).toBe(process.env.EXPECTED_BUILD_ID); +}); + +test('changing search while loading another page does not append old results', async ({ page }) => { + await page.goto('/?phase=primary'); + await expect(page.getByRole('button', { name: 'Load more schools' })).toBeVisible(); + let received!: (route: Route) => void; + const pending = new Promise(resolve => { received = resolve; }); + await page.route('**/api/schools?**', async route => { + if (new URL(route.request().url()).searchParams.get('page') === '2') { + received(route); + return; + } + await route.continue(); + }); + await page.getByRole('button', { name: 'Load more schools' }).click(); + const oldRequest = await pending; + const search = page.getByPlaceholder('School name or postcode').first(); + await search.fill('secondary'); + await search.press('Enter'); + await page.waitForURL(/search=secondary/); + // A cancelled fetch may prevent route fulfilment altogether; either way, + // this deliberately late response must not become part of the new results. + await oldRequest.fulfill({ json: { + schools: [{ urn: 999998, school_name: 'P1 stale result sentinel', phase: 'Primary' }], + total: 2, page: 2, page_size: 1, total_pages: 2, + } }).catch(() => {}); + await expect(page.getByText('P1 stale result sentinel')).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Loading...' })).toHaveCount(0); +}); diff --git a/nextjs-app/Dockerfile b/nextjs-app/Dockerfile index af446c9..21962b5 100644 --- a/nextjs-app/Dockerfile +++ b/nextjs-app/Dockerfile @@ -28,6 +28,10 @@ ENV NODE_ENV=production ARG FASTAPI_URL=http://backend:80/api ENV FASTAPI_URL=${FASTAPI_URL} +ARG BUILD_SHA=development +ARG BUILD_ID=development +RUN node -e 'require("fs").writeFileSync("build-info.json", JSON.stringify({sha:process.argv[1],build_id:process.argv[2]}))' "$BUILD_SHA" "$BUILD_ID" + # Build application RUN npm run build @@ -70,6 +74,12 @@ USER nextjs EXPOSE 3000 # Set environment variables +ARG BUILD_SHA=development +ARG BUILD_ID=development +LABEL io.schoolcompare.build-id=$BUILD_ID +LABEL io.schoolcompare.commit=$BUILD_SHA +COPY --from=builder /app/build-info.json ./build-info.json + ENV PORT=3000 ENV HOSTNAME="0.0.0.0" diff --git a/nextjs-app/__tests__/app/releaseIdentity.test.ts b/nextjs-app/__tests__/app/releaseIdentity.test.ts new file mode 100644 index 0000000..a2476d1 --- /dev/null +++ b/nextjs-app/__tests__/app/releaseIdentity.test.ts @@ -0,0 +1,30 @@ +/** @jest-environment node */ +import { GET } from '@/app/(frontend)/release.json/route'; +import { readFile } from 'node:fs/promises'; + +jest.mock('node:fs/promises', () => ({ readFile: jest.fn() })); +const realFetch = global.fetch; +const identity = { sha: 'a'.repeat(40), build_id: 'b'.repeat(32) }; +beforeEach(() => { + jest.mocked(readFile).mockResolvedValue(JSON.stringify(identity)); + global.fetch = jest.fn(async () => Response.json(identity)); +}); +afterEach(() => { global.fetch = realFetch; jest.resetAllMocks(); }); + +test('reports immutable file identity and backend identity without caching', async () => { + const response = await GET(); + expect(response.status).toBe(200); + expect(response.headers.get('Cache-Control')).toBe('no-store'); + expect(await response.json()).toEqual({ frontend: identity, backend: identity }); + expect(fetch).toHaveBeenCalledWith(expect.stringMatching(/\/api\/release$/), expect.objectContaining({ cache: 'no-store', signal: expect.anything() })); +}); + +test('missing build metadata cannot pass the release gate', async () => { + jest.mocked(readFile).mockRejectedValueOnce(new Error('missing file')); + expect((await GET()).status).toBe(503); +}); + +test('backend failure cannot pass the release gate', async () => { + jest.mocked(fetch).mockResolvedValueOnce(new Response('', { status: 503 })); + expect((await GET()).status).toBe(503); +}); diff --git a/nextjs-app/app/(frontend)/release.json/route.ts b/nextjs-app/app/(frontend)/release.json/route.ts new file mode 100644 index 0000000..9dd7456 --- /dev/null +++ b/nextjs-app/app/(frontend)/release.json/route.ts @@ -0,0 +1,17 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +export async function GET() { + try { + const frontend = JSON.parse(await readFile(path.join(process.cwd(), 'build-info.json'), 'utf8')); + const base = process.env.FASTAPI_URL || 'http://localhost:8000/api'; + const res = await fetch(`${base}/release`, { cache: 'no-store', signal: AbortSignal.timeout(5000) }); + if (!res.ok) throw new Error('Backend identity unavailable'); + return Response.json({ frontend, backend: await res.json() }, { headers: { 'Cache-Control': 'no-store', 'X-Robots-Tag': 'noindex' } }); + } catch { + return Response.json({ detail: 'Release identity unavailable' }, { status: 503, headers: { 'Cache-Control': 'no-store', 'X-Robots-Tag': 'noindex' } }); + } +} diff --git a/pipeline/Dockerfile b/pipeline/Dockerfile index cf16a25..68599d3 100644 --- a/pipeline/Dockerfile +++ b/pipeline/Dockerfile @@ -40,4 +40,9 @@ ENV AIRFLOW_HOME=/opt/airflow ENV AIRFLOW__CORE__DAGS_FOLDER=/opt/pipeline/dags ENV PYTHONPATH=/opt/pipeline +ARG BUILD_SHA=development +ARG BUILD_ID=development +LABEL io.schoolcompare.build-id=$BUILD_ID +LABEL io.schoolcompare.commit=$BUILD_SHA + CMD ["airflow", "api-server"] diff --git a/scripts/ci/release.py b/scripts/ci/release.py new file mode 100644 index 0000000..fd41c62 --- /dev/null +++ b/scripts/ci/release.py @@ -0,0 +1,136 @@ +"""Release identity checks and promotion of the exact digests that passed E2E. + +Uses only the standard library and Docker Buildx. No registry mutation happens +until every image in the set has been resolved and its build labels validated. +""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import time +from urllib.request import Request, urlopen + +COMPONENTS = ('BACKEND', 'FRONTEND', 'PIPELINE') + + +def docker(*args): + return subprocess.check_output(['docker', 'buildx', 'imagetools', *args], text=True).strip() + + +def check_sha(sha): + if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError('Expected a full commit SHA') + return sha + + +def check_digest(digest): + if not re.fullmatch(r'sha256:[0-9a-f]{64}', digest): + raise ValueError('Expected an immutable image digest') + return digest + + +def image_identity(ref): + image = json.loads(docker('inspect', ref, '--format', '{{json .Image}}')) + configs = [image] if 'config' in image else list(image.values()) + identities = set() + for config in configs: + labels = config['config']['Labels'] + identities.add((labels['io.schoolcompare.commit'], labels['io.schoolcompare.build-id'])) + if len(identities) != 1: + raise ValueError('Image platforms disagree about their release identity') + return next(iter(identities)) + + +def resolve_images(sha, verified=False, expected_build_id=None): + check_sha(sha) + refs = [] + build_ids = set() + for component in COMPONENTS: + image = f"{os.environ['REGISTRY']}/{os.environ[component + '_IMAGE_NAME']}" + if verified: + manifest = json.loads(docker('inspect', f'{image}:verified-{sha}', '--format', '{{json .Manifest}}')) + digest = check_digest(manifest['digest']) + else: + digest = check_digest(os.environ[component + '_DIGEST']) + ref = f'{image}@{digest}' + actual_sha, build_id = image_identity(ref) + if actual_sha != sha or not re.fullmatch(r'[0-9a-f]{32}', build_id): + raise ValueError(f'Unrecognised release identity for {component}') + if expected_build_id is not None and build_id != expected_build_id: + raise ValueError(f'Build identity mismatch for {component}') + build_ids.add(build_id) + refs.append((image, ref)) + if len(build_ids) != 1: + raise ValueError('Refusing a mixed image set') + return {'sha': sha, 'build_id': build_ids.pop(), 'images': refs} + + +def verify(sha, build_id): + release = resolve_images(sha, expected_build_id=build_id) + for image, ref in release['images']: + docker('create', '-t', f'{image}:verified-{sha}', ref) + return release + + +def promote(sha): + release = resolve_images(sha, verified=True) + # Resolve all targets first; never discover a missing candidate halfway through. + for image, _ in release['images']: + try: + docker('create', '-t', f'{image}:prod-previous', f'{image}:prod') + except subprocess.CalledProcessError: + print(f'No rollback pointer saved for {image}', flush=True) + for image, ref in release['images']: + docker('create', '-t', f'{image}:prod', ref) + return release + + +def matches(payload, sha, build_id): + return all(payload.get(component) == {'sha': sha, 'build_id': build_id} + for component in ('frontend', 'backend')) + + +def wait(base_url, sha, build_id, timeout): + check_sha(sha) + if not re.fullmatch(r'[0-9a-f]{32}', build_id): + raise ValueError('Missing expected build identity') + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + req = Request(f'{base_url.rstrip("/")}/release.json?check={time.time_ns()}', + headers={'Cache-Control': 'no-cache'}) + with urlopen(req, timeout=min(10, max(.1, deadline - time.monotonic()))) as response: + payload = json.load(response) + if matches(payload, sha, build_id): + print(f'Verified deployed release {sha} / {build_id}') + return + except (OSError, ValueError): + pass + time.sleep(min(5, max(0, deadline - time.monotonic()))) + raise RuntimeError('Deployment did not report the expected frontend/backend release') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['wait', 'verify', 'promote']) + parser.add_argument('--timeout', type=float, default=300) + parser.add_argument('--release', type=Path) + parser.add_argument('--output', type=Path) + args = parser.parse_args() + identity = json.loads(args.release.read_text()) if args.release else { + 'sha': os.environ.get('EXPECTED_SHA', ''), + 'build_id': os.environ.get('EXPECTED_BUILD_ID', ''), + } + if args.action == 'wait': + wait(os.environ['BASE_URL'], identity['sha'], identity['build_id'], args.timeout) + return + result = (verify(identity['sha'], identity['build_id']) if args.action == 'verify' + else promote(identity['sha'])) + if args.output: + args.output.write_text(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/scripts/ci/tests/test_release.py b/scripts/ci/tests/test_release.py new file mode 100644 index 0000000..c98af86 --- /dev/null +++ b/scripts/ci/tests/test_release.py @@ -0,0 +1,65 @@ +import json +from unittest.mock import Mock +import pytest +from scripts.ci import release + +SHA = 'a' * 40 +BUILD = 'b' * 32 +DIGESTS = ['sha256:' + c * 64 for c in '123'] + + +@pytest.fixture +def docker(monkeypatch): + monkeypatch.setenv('REGISTRY', 'registry.example') + refs = {} + for component, digest in zip(release.COMPONENTS, DIGESTS): + monkeypatch.setenv(component + '_IMAGE_NAME', component.lower()) + monkeypatch.setenv(component + '_DIGEST', digest) + refs[f'registry.example/{component.lower()}'] = digest + def run(*args): + if args[0] == 'create': return '' + if args[-1] == '{{json .Manifest}}': + return json.dumps({'digest': refs[args[1].split(':')[0]]}) + return json.dumps({'config': {'Labels': {'io.schoolcompare.commit': SHA, + 'io.schoolcompare.build-id': BUILD}}}) + mock = Mock(side_effect=run) + monkeypatch.setattr(release, 'docker', mock) + return mock + + +def test_wrong_deployed_build_is_rejected_even_at_same_commit(): + assert not release.matches({'frontend': {'sha': SHA, 'build_id': BUILD}, + 'backend': {'sha': SHA, 'build_id': 'c' * 32}}, SHA, BUILD) + assert release.matches({c: {'sha': SHA, 'build_id': BUILD} for c in ('frontend', 'backend')}, SHA, BUILD) + + +def test_verification_tags_the_captured_digests(docker): + release.verify(SHA, BUILD) + creates = [c.args for c in docker.call_args_list if c.args[0] == 'create'] + assert len(creates) == 3 + for call, digest in zip(creates, DIGESTS): + assert call[-1].endswith('@' + digest) + assert call[2].endswith(':verified-' + SHA) + + +def test_promotion_resolves_all_verified_images_before_mutation(docker): + result = release.promote(SHA) + assert result['build_id'] == BUILD + calls = [c.args for c in docker.call_args_list] + first_write = next(i for i, c in enumerate(calls) if c[0] == 'create') + assert first_write == 6 # each of three candidates needs manifest + config + assert all(c[-1].endswith('@' + d) for c, d in zip(calls[-3:], DIGESTS)) + + +def test_mixed_builds_fail_before_any_tag_is_changed(docker, monkeypatch): + identities = iter([(SHA, BUILD), (SHA, 'c' * 32), (SHA, BUILD)]) + monkeypatch.setattr(release, 'image_identity', lambda _: next(identities)) + with pytest.raises(ValueError, match='mixed'): + release.promote(SHA) + assert not any(c.args[0] == 'create' for c in docker.call_args_list) + + +def test_missing_candidate_fails_before_any_tag_is_changed(docker): + docker.side_effect = RuntimeError('missing verified tag') + with pytest.raises(RuntimeError): release.promote(SHA) + assert not any(c.args[0] == 'create' for c in docker.call_args_list) diff --git a/scripts/ci/tests/test_workflows.py b/scripts/ci/tests/test_workflows.py new file mode 100644 index 0000000..adb3bae --- /dev/null +++ b/scripts/ci/tests/test_workflows.py @@ -0,0 +1,34 @@ +"""Check the dependency graph that ties tested digests to deployable images.""" +from pathlib import Path +import yaml + +ROOT = Path(__file__).resolve().parents[3] + + +def test_staging_verifies_identity_before_and_after_journeys(): + workflow = yaml.safe_load((ROOT / '.gitea/workflows/deploy.yml').read_text()) + assert workflow['concurrency'] == {'group': 'staging-release', 'cancel-in-progress': False} + jobs = workflow['jobs'] + for component in ('backend', 'frontend', 'pipeline'): + job = jobs['build-' + component] + assert 'prepare' in job['needs'] + assert job['outputs']['digest'] == '${{ steps.build.outputs.digest }}' + build = next(step for step in job['steps'] if step.get('id') == 'build') + assert 'BUILD_ID=${{ needs.prepare.outputs.build_id }}' in build['with']['build-args'] + steps = jobs['e2e-staging']['steps'] + runs = [step.get('run', '') for step in steps] + test = runs.index('npx playwright test') + assert 'release.py wait' in runs[test - 1] + assert 'release.py wait' in runs[test + 1] + assert 'release.py verify' in runs[-1] + for component in ('backend', 'frontend', 'pipeline'): + assert 'build-' + component in jobs['e2e-staging']['needs'] + assert component.upper() + '_DIGEST' in steps[-1]['env'] + + +def test_promotion_uses_verified_digest_resolver_and_build_identity_poll(): + workflow = yaml.safe_load((ROOT / '.gitea/workflows/promote.yml').read_text()) + steps = workflow['jobs']['promote-prod']['steps'] + runs = [step.get('run', '') for step in steps] + assert 'python3 scripts/ci/release.py promote --output release.json' in runs + assert runs[-1] == 'python3 scripts/ci/release.py wait --release release.json'