diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 31bb6ea..f24cb71 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -5,6 +5,11 @@ on: branches: - main +# Serialise the entire build/deploy/test cycle: no other run can move staging tags. +concurrency: + group: staging-release + cancel-in-progress: false + env: REGISTRY: privaterepo.sitaru.org BACKEND_IMAGE_NAME: ${{ gitea.repository }}-backend @@ -12,7 +17,18 @@ env: PIPELINE_IMAGE_NAME: ${{ gitea.repository }}-pipeline jobs: + prepare: + runs-on: ubuntu-latest + outputs: + build_id: ${{ steps.identity.outputs.build_id }} + steps: + - id: identity + run: python3 -c 'import uuid; print("build_id=" + uuid.uuid4().hex)' >> "$GITHUB_OUTPUT" + build-backend: + needs: [prepare] + outputs: + digest: ${{ steps.build.outputs.digest }} name: Build Backend (FastAPI) runs-on: ubuntu-latest steps: @@ -46,17 +62,24 @@ jobs: type=raw,value=staging - name: Build and push Backend Docker image + id: build uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true + build-args: | + BUILD_SHA=${{ gitea.sha }} + BUILD_ID=${{ needs.prepare.outputs.build_id }} tags: ${{ steps.meta-backend.outputs.tags }} labels: ${{ steps.meta-backend.outputs.labels }} cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:buildcache,mode=max build-frontend: + needs: [prepare] + outputs: + digest: ${{ steps.build.outputs.digest }} name: Build Frontend (Next.js) runs-on: ubuntu-latest steps: @@ -90,18 +113,23 @@ jobs: type=raw,value=staging - name: Build and push Frontend Docker image + id: build uses: docker/build-push-action@v5 with: context: ./nextjs-app file: ./nextjs-app/Dockerfile push: true + build-args: | + BUILD_SHA=${{ gitea.sha }} + BUILD_ID=${{ needs.prepare.outputs.build_id }} tags: ${{ steps.meta-frontend.outputs.tags }} labels: ${{ steps.meta-frontend.outputs.labels }} - build-args: | - FASTAPI_URL=http://backend:80/api # Cache disabled due to registry size limits build-pipeline: + needs: [prepare] + outputs: + digest: ${{ steps.build.outputs.digest }} name: Build Pipeline (Meltano + dbt + Airflow) runs-on: ubuntu-latest steps: @@ -135,11 +163,15 @@ jobs: type=raw,value=staging - name: Build and push Pipeline Docker image + id: build uses: docker/build-push-action@v5 with: context: ./pipeline file: ./pipeline/Dockerfile push: true + build-args: | + BUILD_SHA=${{ gitea.sha }} + BUILD_ID=${{ needs.prepare.outputs.build_id }} tags: ${{ steps.meta-pipeline.outputs.tags }} labels: ${{ steps.meta-pipeline.outputs.labels }} cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PIPELINE_IMAGE_NAME }}:buildcache @@ -148,30 +180,23 @@ jobs: deploy-staging: name: Deploy to Staging runs-on: ubuntu-latest - needs: [build-backend, build-frontend, build-pipeline] + needs: [prepare, build-backend, build-frontend, build-pipeline] steps: - name: Trigger staging stack update run: curl -fsSk -X POST "${{ secrets.PORTAINER_STAGING_WEBHOOK }}" - - name: Wait for staging to become healthy - run: | - echo "Polling ${STAGING_BASE_URL} for up to 5 minutes..." - for i in $(seq 1 60); do - if curl -fsS -o /dev/null --max-time 10 "${STAGING_BASE_URL}/"; then - echo "Staging is up (attempt $i)" - exit 0 - fi - sleep 5 - done - echo "Staging did not become healthy in time" >&2 - exit 1 + - uses: actions/checkout@v4 + - name: Verify deployed release identity + run: python3 scripts/ci/release.py wait env: - STAGING_BASE_URL: ${{ secrets.STAGING_BASE_URL }} + BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} e2e-staging: name: E2E Journeys against Staging runs-on: ubuntu-latest - needs: [deploy-staging] + needs: [prepare, deploy-staging, build-backend, build-frontend, build-pipeline] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -187,11 +212,42 @@ jobs: npm ci npx playwright install --with-deps chromium + - name: Verify release before journeys + run: python3 scripts/ci/release.py wait --timeout 10 + env: + BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + - name: Run E2E journeys working-directory: e2e run: npx playwright test env: BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + + - name: Verify release after journeys + run: python3 scripts/ci/release.py wait --timeout 10 + env: + BASE_URL: ${{ secrets.STAGING_BASE_URL }} + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ gitea.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Mark tested image digests as verified + run: python3 scripts/ci/release.py verify + env: + EXPECTED_SHA: ${{ gitea.sha }} + EXPECTED_BUILD_ID: ${{ needs.prepare.outputs.build_id }} + BACKEND_DIGEST: ${{ needs.build-backend.outputs.digest }} + FRONTEND_DIGEST: ${{ needs.build-frontend.outputs.digest }} + PIPELINE_DIGEST: ${{ needs.build-pipeline.outputs.digest }} # Production deployment is a second, manual approval: see promote.yml # ("Promote to Production (manual)") and docs/DEPLOY.md. diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index b7d3a11..09a843d 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -68,13 +68,13 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install -r requirements.txt pytest "httpx<0.28" + run: pip install -r requirements.txt pytest "httpx<0.28" pyyaml - name: Import smoke test run: python -c "from backend.app import app; print('backend imports OK')" - name: Backend unit tests - run: python -m pytest backend/tests -q + run: python -m pytest backend/tests pipeline/tests scripts/ci/tests -q build-backend: name: Build Backend (no push) diff --git a/.gitea/workflows/promote.yml b/.gitea/workflows/promote.yml index 777530c..35652bb 100644 --- a/.gitea/workflows/promote.yml +++ b/.gitea/workflows/promote.yml @@ -97,33 +97,15 @@ jobs: username: ${{ gitea.actor }} password: ${{ secrets.REGISTRY_TOKEN }} - - name: Retag approved images as prod (keeping rollback pointer) - run: | - SHORT_SHA="${{ steps.resolve.outputs.short }}" - for IMAGE in \ - "${REGISTRY}/${BACKEND_IMAGE_NAME}" \ - "${REGISTRY}/${FRONTEND_IMAGE_NAME}" \ - "${REGISTRY}/${PIPELINE_IMAGE_NAME}"; do - # Keep a rollback pointer before moving :prod - docker buildx imagetools create -t "${IMAGE}:prod-previous" "${IMAGE}:prod" || true - docker buildx imagetools create -t "${IMAGE}:prod" "${IMAGE}:${SHORT_SHA}" - echo "Promoted ${IMAGE}:${SHORT_SHA} -> :prod" - done + - name: Resolve verified digests and promote the complete image set + run: python3 scripts/ci/release.py promote --output release.json + env: + EXPECTED_SHA: ${{ steps.resolve.outputs.full }} - name: Trigger production stack update run: curl -fsSk -X POST "${{ secrets.PORTAINER_PROD_WEBHOOK }}" - - name: Wait for production to become healthy - run: | - echo "Polling ${PROD_BASE_URL} for up to 5 minutes..." - for i in $(seq 1 60); do - if curl -fsS -o /dev/null --max-time 10 "${PROD_BASE_URL}/"; then - echo "Production is up (attempt $i)" - exit 0 - fi - sleep 5 - done - echo "Production did not become healthy in time" >&2 - exit 1 + - name: Verify production release identity + run: python3 scripts/ci/release.py wait --release release.json env: - PROD_BASE_URL: ${{ secrets.PROD_BASE_URL }} + BASE_URL: ${{ secrets.PROD_BASE_URL }} diff --git a/Dockerfile b/Dockerfile index 6b9011f..c048914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,12 @@ RUN pip install --no-cache-dir -r requirements.txt COPY backend/ ./backend/ COPY scripts/ ./scripts/ +ARG BUILD_SHA=development +ARG BUILD_ID=development +LABEL io.schoolcompare.build-id=$BUILD_ID +LABEL io.schoolcompare.commit=$BUILD_SHA +RUN python -c 'import json,sys; open("backend/build-info.json", "w").write(json.dumps({"sha":sys.argv[1],"build_id":sys.argv[2]}))' "$BUILD_SHA" "$BUILD_ID" + # Expose the application port EXPOSE 80 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4ea42cd..a00995e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -80,9 +80,11 @@ API types. `payload-types.ts` and the Payload import map are generated artifacts 1. Airflow DAGs extract and validate source data, then run selected dbt builds. 2. Relevant DAGs rebuild Typesense and swap the `schools` alias. -3. They call `POST /api/admin/reload` with `X-API-Key` to refresh school DataFrames. -4. A separate weekly sitemap DAG calls `POST /api/admin/regenerate-sitemap`, - rebuilding places and sitemaps. +3. They call `POST /api/admin/reload` with `X-API-Key`. It builds and validates + replacement DataFrames, places, reverse membership and sitemaps off the request + loop, then publishes them together. Failure returns 503 and preserves live data. +4. A separate weekly sitemap DAG can regenerate the derived publication from the + current DataFrame without clearing the live registry first. GIAS is scheduled daily, Ofsted monthly, and annual datasets are manually triggered. The DAG definitions are authoritative for selectors and dependencies. @@ -92,16 +94,25 @@ backend HTTP Cache-Control/ETags, Next.js fetch/page revalidation, and browser o shared HTTP caches where configured. Place fetches request a one-week revalidation interval. HTTP ETags are computed after route execution, not before database work. -Known limitations: reload clears the old DataFrames before verifying replacement -data; places/sitemaps refresh separately; Next.js caches are not explicitly purged -by the pipeline; Typesense import results are not validated before alias publication. -Do not describe this sequence as an atomic dataset release. These are follow-up -reliability tasks, not changes implemented by the documentation cleanup. +Typesense publication validates every import response and the final document +count before switching aliases. A session-scoped PostgreSQL advisory lock +serialises index reads/publication across DAGs. The previous collection remains +available for rollback; old unaliased collections are pruned after success. +Failed drafts are retained until a later successful cleanup, because an uncertain +alias-update response must never cause deletion of a potentially live index. + +The backend snapshot swap is process-local and assumes the current single-worker +deployment. It is not an atomic transaction spanning PostgreSQL marts, Typesense +and Next.js caches. Next.js caches are not explicitly purged by the pipeline. +School search retrieves every Typesense candidate before applying API filters; +only a dependency failure invokes substring fallback, not a valid empty match set. ## Deployment references See [DEPLOY.md](DEPLOY.md). PR checks include frontend typechecking/tests, backend unit tests, image builds and AI review. Staging journeys run after merging. -Production promotion retags a selected commit's images. Current health polling -checks HTTP success, not the deployed commit identity; overlapping staging runs -remain a release-verification concern. +Staging runs are serialised across builds, deployment and E2E. Build-stamped +frontend/backend identities are checked before and after journeys. Only then are +the captured image digests marked verified. Promotion resolves and validates the +complete verified image set before retagging production. See the runbook for +first-rollout requirements and remaining integration checks. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 2cff1b8..63e85e0 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -19,8 +19,9 @@ PR checks (.gitea/workflows/pr-checks.yml) ▼ Stage pipeline (.gitea/workflows/deploy.yml) — automatic 1. build & push images → tags sha-, staging - 2. staging Portainer webhook → wait for staging health + 2. staging Portainer webhook → verify frontend/backend SHA + build ID 3. Playwright E2E journeys against staging ← gate before human testing + 4. verify identity again; tag tested digests verified- ▼ Manual testing on staging (stx.schoolcompare.co.uk) │ Actions → "Promote to Production (manual)" ← approval #2 @@ -28,14 +29,15 @@ Manual testing on staging (stx.schoolcompare.co.uk) Promote pipeline (.gitea/workflows/promote.yml) — manual dispatch 1. resolve target sha (input, or latest main if empty) 2. REFUSE unless that commit's "E2E Journeys against Staging" status is green - 3. retag sha- → :prod (same bytes — build once, promote the image) + 3. resolve verified- digests, validate labels, retag digests → :prod previous :prod saved as :prod-previous - 4. prod Portainer webhook → wait for prod health + 4. prod Portainer webhook → verify expected SHA + build ID ``` Key principle: **build once, promote the exact image**. Production pins `:prod`, which only moves when a human runs the promote workflow — and the workflow -only accepts commits that passed the staging E2E gate. Nothing tags `:latest` +only accepts commits that passed the staging E2E gate and have a complete verified +image set. Nothing tags `:latest` anymore. ## Branch & PR workflow @@ -256,3 +258,41 @@ how long any feature is exposed to this. If `UNLEASH_URL` is unset, every flag is `False` and no connection is attempted. That is the correct behaviour for local development and CI, and it means the test suites need no flag server. + +## Release identity and the P1 reliability gate + +Every staging run creates a random build ID before building its three images. +Each image carries the commit and build ID as labels. Frontend/backend images +also contain a build-time JSON file; environment overrides cannot rewrite it. +`/release.json` returns both identities with `Cache-Control: no-store`. It fails +with 503 when either identity cannot be read. FastAPI's internal endpoint is +`/api/release`. + +The entire staging workflow shares one concurrency group, with cancellation +disabled. This needs Gitea 1.26 or newer, where workflow concurrency is supported +([release notes](https://blog.gitea.com/release-of-1.26.0/)); the configured server +reported 1.27.3 during this change. Do not run the workflow on an older server +that ignores the concurrency key. Manual deployments outside this workflow must +also avoid changing staging during journeys. + +The gate checks both identities before and after Playwright. It then validates +labels on the captured build output digests and tags them `verified-`. +The manual promotion script resolves all three verified tags to immutable digests +and confirms one matching commit/build ID before moving any `:prod` tag. It polls +production for that same identity using a locally saved release manifest. +A registry error can still interrupt the three tag writes; the Portainer webhook +only runs after successful promotion, and rerunning promotion resolves the full +verified set again. There is no cross-registry atomic tag transaction. + +**First rollout:** old green commits without verified tags/build identities are +not promotable through this gate. Build and test a commit containing the new +workflow first. The release route must be reachable through the configured +`STAGING_BASE_URL`/`PROD_BASE_URL`; it deliberately avoids the public staging +`/api` proxy limitation. No new deployment secret is required. + +`scripts/ci/release.py` implements identity polling and digest verification. +Its mocked tests run in PR checks alongside backend and index-publication tests. +The new Playwright journeys also check deployed identity and stale pagination. +Local unit checks do not validate registry credentials, Portainer behaviour, +proxy routing or a deployed image; those require the staging run. Production +promotion remains a separate human action. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index f1ebb0c..613cc85 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -42,8 +42,8 @@ From the repository root, using an available Python 3.11 or 3.12 interpreter: ```sh python3.11 -m venv /tmp/schoolcompare-backend-venv -/tmp/schoolcompare-backend-venv/bin/python -m pip install -r requirements.txt pytest 'httpx<0.28' -/tmp/schoolcompare-backend-venv/bin/python -m pytest backend/tests -q +/tmp/schoolcompare-backend-venv/bin/python -m pip install -r requirements.txt pytest 'httpx<0.28' pyyaml +/tmp/schoolcompare-backend-venv/bin/python -m pytest backend/tests pipeline/tests scripts/ci/tests -q ``` Substitute `python3.12` if matching PR CI. The test dependencies above match the diff --git a/e2e/tests/reliability.spec.ts b/e2e/tests/reliability.spec.ts new file mode 100644 index 0000000..31a2d4a --- /dev/null +++ b/e2e/tests/reliability.spec.ts @@ -0,0 +1,41 @@ +import { test, expect, Route } from '@playwright/test'; + +test('the deployed frontend and backend report the tested build', async ({ request }) => { + const response = await request.get('/release.json'); + expect(response.ok()).toBeTruthy(); + expect(response.headers()['cache-control']).toContain('no-store'); + const identity = await response.json(); + expect(identity.frontend).toEqual(identity.backend); + expect(identity.frontend.sha).toMatch(/^[a-f0-9]{40}$/); + expect(identity.frontend.build_id).toMatch(/^[a-f0-9]{32}$/); + if (process.env.EXPECTED_SHA) expect(identity.frontend.sha).toBe(process.env.EXPECTED_SHA); + if (process.env.EXPECTED_BUILD_ID) expect(identity.frontend.build_id).toBe(process.env.EXPECTED_BUILD_ID); +}); + +test('changing search while loading another page does not append old results', async ({ page }) => { + await page.goto('/?phase=primary'); + await expect(page.getByRole('button', { name: 'Load more schools' })).toBeVisible(); + let received!: (route: Route) => void; + const pending = new Promise(resolve => { received = resolve; }); + await page.route('**/api/schools?**', async route => { + if (new URL(route.request().url()).searchParams.get('page') === '2') { + received(route); + return; + } + await route.continue(); + }); + await page.getByRole('button', { name: 'Load more schools' }).click(); + const oldRequest = await pending; + const search = page.getByPlaceholder('School name or postcode').first(); + await search.fill('secondary'); + await search.press('Enter'); + await page.waitForURL(/search=secondary/); + // A cancelled fetch may prevent route fulfilment altogether; either way, + // this deliberately late response must not become part of the new results. + await oldRequest.fulfill({ json: { + schools: [{ urn: 999998, school_name: 'P1 stale result sentinel', phase: 'Primary' }], + total: 2, page: 2, page_size: 1, total_pages: 2, + } }).catch(() => {}); + await expect(page.getByText('P1 stale result sentinel')).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Loading...' })).toHaveCount(0); +}); diff --git a/nextjs-app/Dockerfile b/nextjs-app/Dockerfile index af446c9..21962b5 100644 --- a/nextjs-app/Dockerfile +++ b/nextjs-app/Dockerfile @@ -28,6 +28,10 @@ ENV NODE_ENV=production ARG FASTAPI_URL=http://backend:80/api ENV FASTAPI_URL=${FASTAPI_URL} +ARG BUILD_SHA=development +ARG BUILD_ID=development +RUN node -e 'require("fs").writeFileSync("build-info.json", JSON.stringify({sha:process.argv[1],build_id:process.argv[2]}))' "$BUILD_SHA" "$BUILD_ID" + # Build application RUN npm run build @@ -70,6 +74,12 @@ USER nextjs EXPOSE 3000 # Set environment variables +ARG BUILD_SHA=development +ARG BUILD_ID=development +LABEL io.schoolcompare.build-id=$BUILD_ID +LABEL io.schoolcompare.commit=$BUILD_SHA +COPY --from=builder /app/build-info.json ./build-info.json + ENV PORT=3000 ENV HOSTNAME="0.0.0.0" diff --git a/nextjs-app/__tests__/app/releaseIdentity.test.ts b/nextjs-app/__tests__/app/releaseIdentity.test.ts new file mode 100644 index 0000000..a2476d1 --- /dev/null +++ b/nextjs-app/__tests__/app/releaseIdentity.test.ts @@ -0,0 +1,30 @@ +/** @jest-environment node */ +import { GET } from '@/app/(frontend)/release.json/route'; +import { readFile } from 'node:fs/promises'; + +jest.mock('node:fs/promises', () => ({ readFile: jest.fn() })); +const realFetch = global.fetch; +const identity = { sha: 'a'.repeat(40), build_id: 'b'.repeat(32) }; +beforeEach(() => { + jest.mocked(readFile).mockResolvedValue(JSON.stringify(identity)); + global.fetch = jest.fn(async () => Response.json(identity)); +}); +afterEach(() => { global.fetch = realFetch; jest.resetAllMocks(); }); + +test('reports immutable file identity and backend identity without caching', async () => { + const response = await GET(); + expect(response.status).toBe(200); + expect(response.headers.get('Cache-Control')).toBe('no-store'); + expect(await response.json()).toEqual({ frontend: identity, backend: identity }); + expect(fetch).toHaveBeenCalledWith(expect.stringMatching(/\/api\/release$/), expect.objectContaining({ cache: 'no-store', signal: expect.anything() })); +}); + +test('missing build metadata cannot pass the release gate', async () => { + jest.mocked(readFile).mockRejectedValueOnce(new Error('missing file')); + expect((await GET()).status).toBe(503); +}); + +test('backend failure cannot pass the release gate', async () => { + jest.mocked(fetch).mockResolvedValueOnce(new Response('', { status: 503 })); + expect((await GET()).status).toBe(503); +}); diff --git a/nextjs-app/app/(frontend)/release.json/route.ts b/nextjs-app/app/(frontend)/release.json/route.ts new file mode 100644 index 0000000..9dd7456 --- /dev/null +++ b/nextjs-app/app/(frontend)/release.json/route.ts @@ -0,0 +1,17 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +export async function GET() { + try { + const frontend = JSON.parse(await readFile(path.join(process.cwd(), 'build-info.json'), 'utf8')); + const base = process.env.FASTAPI_URL || 'http://localhost:8000/api'; + const res = await fetch(`${base}/release`, { cache: 'no-store', signal: AbortSignal.timeout(5000) }); + if (!res.ok) throw new Error('Backend identity unavailable'); + return Response.json({ frontend, backend: await res.json() }, { headers: { 'Cache-Control': 'no-store', 'X-Robots-Tag': 'noindex' } }); + } catch { + return Response.json({ detail: 'Release identity unavailable' }, { status: 503, headers: { 'Cache-Control': 'no-store', 'X-Robots-Tag': 'noindex' } }); + } +} diff --git a/pipeline/Dockerfile b/pipeline/Dockerfile index cf16a25..68599d3 100644 --- a/pipeline/Dockerfile +++ b/pipeline/Dockerfile @@ -40,4 +40,9 @@ ENV AIRFLOW_HOME=/opt/airflow ENV AIRFLOW__CORE__DAGS_FOLDER=/opt/pipeline/dags ENV PYTHONPATH=/opt/pipeline +ARG BUILD_SHA=development +ARG BUILD_ID=development +LABEL io.schoolcompare.build-id=$BUILD_ID +LABEL io.schoolcompare.commit=$BUILD_SHA + CMD ["airflow", "api-server"] diff --git a/scripts/ci/release.py b/scripts/ci/release.py new file mode 100644 index 0000000..fd41c62 --- /dev/null +++ b/scripts/ci/release.py @@ -0,0 +1,136 @@ +"""Release identity checks and promotion of the exact digests that passed E2E. + +Uses only the standard library and Docker Buildx. No registry mutation happens +until every image in the set has been resolved and its build labels validated. +""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import time +from urllib.request import Request, urlopen + +COMPONENTS = ('BACKEND', 'FRONTEND', 'PIPELINE') + + +def docker(*args): + return subprocess.check_output(['docker', 'buildx', 'imagetools', *args], text=True).strip() + + +def check_sha(sha): + if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError('Expected a full commit SHA') + return sha + + +def check_digest(digest): + if not re.fullmatch(r'sha256:[0-9a-f]{64}', digest): + raise ValueError('Expected an immutable image digest') + return digest + + +def image_identity(ref): + image = json.loads(docker('inspect', ref, '--format', '{{json .Image}}')) + configs = [image] if 'config' in image else list(image.values()) + identities = set() + for config in configs: + labels = config['config']['Labels'] + identities.add((labels['io.schoolcompare.commit'], labels['io.schoolcompare.build-id'])) + if len(identities) != 1: + raise ValueError('Image platforms disagree about their release identity') + return next(iter(identities)) + + +def resolve_images(sha, verified=False, expected_build_id=None): + check_sha(sha) + refs = [] + build_ids = set() + for component in COMPONENTS: + image = f"{os.environ['REGISTRY']}/{os.environ[component + '_IMAGE_NAME']}" + if verified: + manifest = json.loads(docker('inspect', f'{image}:verified-{sha}', '--format', '{{json .Manifest}}')) + digest = check_digest(manifest['digest']) + else: + digest = check_digest(os.environ[component + '_DIGEST']) + ref = f'{image}@{digest}' + actual_sha, build_id = image_identity(ref) + if actual_sha != sha or not re.fullmatch(r'[0-9a-f]{32}', build_id): + raise ValueError(f'Unrecognised release identity for {component}') + if expected_build_id is not None and build_id != expected_build_id: + raise ValueError(f'Build identity mismatch for {component}') + build_ids.add(build_id) + refs.append((image, ref)) + if len(build_ids) != 1: + raise ValueError('Refusing a mixed image set') + return {'sha': sha, 'build_id': build_ids.pop(), 'images': refs} + + +def verify(sha, build_id): + release = resolve_images(sha, expected_build_id=build_id) + for image, ref in release['images']: + docker('create', '-t', f'{image}:verified-{sha}', ref) + return release + + +def promote(sha): + release = resolve_images(sha, verified=True) + # Resolve all targets first; never discover a missing candidate halfway through. + for image, _ in release['images']: + try: + docker('create', '-t', f'{image}:prod-previous', f'{image}:prod') + except subprocess.CalledProcessError: + print(f'No rollback pointer saved for {image}', flush=True) + for image, ref in release['images']: + docker('create', '-t', f'{image}:prod', ref) + return release + + +def matches(payload, sha, build_id): + return all(payload.get(component) == {'sha': sha, 'build_id': build_id} + for component in ('frontend', 'backend')) + + +def wait(base_url, sha, build_id, timeout): + check_sha(sha) + if not re.fullmatch(r'[0-9a-f]{32}', build_id): + raise ValueError('Missing expected build identity') + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + req = Request(f'{base_url.rstrip("/")}/release.json?check={time.time_ns()}', + headers={'Cache-Control': 'no-cache'}) + with urlopen(req, timeout=min(10, max(.1, deadline - time.monotonic()))) as response: + payload = json.load(response) + if matches(payload, sha, build_id): + print(f'Verified deployed release {sha} / {build_id}') + return + except (OSError, ValueError): + pass + time.sleep(min(5, max(0, deadline - time.monotonic()))) + raise RuntimeError('Deployment did not report the expected frontend/backend release') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['wait', 'verify', 'promote']) + parser.add_argument('--timeout', type=float, default=300) + parser.add_argument('--release', type=Path) + parser.add_argument('--output', type=Path) + args = parser.parse_args() + identity = json.loads(args.release.read_text()) if args.release else { + 'sha': os.environ.get('EXPECTED_SHA', ''), + 'build_id': os.environ.get('EXPECTED_BUILD_ID', ''), + } + if args.action == 'wait': + wait(os.environ['BASE_URL'], identity['sha'], identity['build_id'], args.timeout) + return + result = (verify(identity['sha'], identity['build_id']) if args.action == 'verify' + else promote(identity['sha'])) + if args.output: + args.output.write_text(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/scripts/ci/tests/test_release.py b/scripts/ci/tests/test_release.py new file mode 100644 index 0000000..c98af86 --- /dev/null +++ b/scripts/ci/tests/test_release.py @@ -0,0 +1,65 @@ +import json +from unittest.mock import Mock +import pytest +from scripts.ci import release + +SHA = 'a' * 40 +BUILD = 'b' * 32 +DIGESTS = ['sha256:' + c * 64 for c in '123'] + + +@pytest.fixture +def docker(monkeypatch): + monkeypatch.setenv('REGISTRY', 'registry.example') + refs = {} + for component, digest in zip(release.COMPONENTS, DIGESTS): + monkeypatch.setenv(component + '_IMAGE_NAME', component.lower()) + monkeypatch.setenv(component + '_DIGEST', digest) + refs[f'registry.example/{component.lower()}'] = digest + def run(*args): + if args[0] == 'create': return '' + if args[-1] == '{{json .Manifest}}': + return json.dumps({'digest': refs[args[1].split(':')[0]]}) + return json.dumps({'config': {'Labels': {'io.schoolcompare.commit': SHA, + 'io.schoolcompare.build-id': BUILD}}}) + mock = Mock(side_effect=run) + monkeypatch.setattr(release, 'docker', mock) + return mock + + +def test_wrong_deployed_build_is_rejected_even_at_same_commit(): + assert not release.matches({'frontend': {'sha': SHA, 'build_id': BUILD}, + 'backend': {'sha': SHA, 'build_id': 'c' * 32}}, SHA, BUILD) + assert release.matches({c: {'sha': SHA, 'build_id': BUILD} for c in ('frontend', 'backend')}, SHA, BUILD) + + +def test_verification_tags_the_captured_digests(docker): + release.verify(SHA, BUILD) + creates = [c.args for c in docker.call_args_list if c.args[0] == 'create'] + assert len(creates) == 3 + for call, digest in zip(creates, DIGESTS): + assert call[-1].endswith('@' + digest) + assert call[2].endswith(':verified-' + SHA) + + +def test_promotion_resolves_all_verified_images_before_mutation(docker): + result = release.promote(SHA) + assert result['build_id'] == BUILD + calls = [c.args for c in docker.call_args_list] + first_write = next(i for i, c in enumerate(calls) if c[0] == 'create') + assert first_write == 6 # each of three candidates needs manifest + config + assert all(c[-1].endswith('@' + d) for c, d in zip(calls[-3:], DIGESTS)) + + +def test_mixed_builds_fail_before_any_tag_is_changed(docker, monkeypatch): + identities = iter([(SHA, BUILD), (SHA, 'c' * 32), (SHA, BUILD)]) + monkeypatch.setattr(release, 'image_identity', lambda _: next(identities)) + with pytest.raises(ValueError, match='mixed'): + release.promote(SHA) + assert not any(c.args[0] == 'create' for c in docker.call_args_list) + + +def test_missing_candidate_fails_before_any_tag_is_changed(docker): + docker.side_effect = RuntimeError('missing verified tag') + with pytest.raises(RuntimeError): release.promote(SHA) + assert not any(c.args[0] == 'create' for c in docker.call_args_list) diff --git a/scripts/ci/tests/test_workflows.py b/scripts/ci/tests/test_workflows.py new file mode 100644 index 0000000..adb3bae --- /dev/null +++ b/scripts/ci/tests/test_workflows.py @@ -0,0 +1,34 @@ +"""Check the dependency graph that ties tested digests to deployable images.""" +from pathlib import Path +import yaml + +ROOT = Path(__file__).resolve().parents[3] + + +def test_staging_verifies_identity_before_and_after_journeys(): + workflow = yaml.safe_load((ROOT / '.gitea/workflows/deploy.yml').read_text()) + assert workflow['concurrency'] == {'group': 'staging-release', 'cancel-in-progress': False} + jobs = workflow['jobs'] + for component in ('backend', 'frontend', 'pipeline'): + job = jobs['build-' + component] + assert 'prepare' in job['needs'] + assert job['outputs']['digest'] == '${{ steps.build.outputs.digest }}' + build = next(step for step in job['steps'] if step.get('id') == 'build') + assert 'BUILD_ID=${{ needs.prepare.outputs.build_id }}' in build['with']['build-args'] + steps = jobs['e2e-staging']['steps'] + runs = [step.get('run', '') for step in steps] + test = runs.index('npx playwright test') + assert 'release.py wait' in runs[test - 1] + assert 'release.py wait' in runs[test + 1] + assert 'release.py verify' in runs[-1] + for component in ('backend', 'frontend', 'pipeline'): + assert 'build-' + component in jobs['e2e-staging']['needs'] + assert component.upper() + '_DIGEST' in steps[-1]['env'] + + +def test_promotion_uses_verified_digest_resolver_and_build_identity_poll(): + workflow = yaml.safe_load((ROOT / '.gitea/workflows/promote.yml').read_text()) + steps = workflow['jobs']['promote-prod']['steps'] + runs = [step.get('run', '') for step in steps] + assert 'python3 scripts/ci/release.py promote --output release.json' in runs + assert runs[-1] == 'python3 scripts/ci/release.py wait --release release.json'