From 1fc1e07d219686a1ab5027454c4e2ec15074ffcf Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 20 Aug 2026 23:21:58 +0100 Subject: [PATCH 1/2] fix(seo): keep staging out of the search index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staging serves the same image as production off stx., with robots.txt saying Allow: / and no noindex — a fully crawlable duplicate of the site. Nothing appears indexed today, most likely because the pages canonicalise across to production, but that is a side effect rather than a control. X-Robots-Tag, not a robots.txt Disallow. Disallow blocks crawling, which is not the same as blocking indexing: a disallowed URL can still be indexed from external links, and blocking the crawl means Google never fetches the page and so never sees a noindex at all. Staging stays crawlable and answers noindex. Matched on the staging host explicitly rather than 'any host that is not production'. The inverted form would cover future environments automatically, but its failure mode is deindexing production if the Host header ever arrives rewritten by a proxy — which cannot be verified from here. This form's failure mode is a new environment being indexable until someone adds it, which is recoverable. Any new non-production hostname must be added. The journeys only ever run against staging (deploy.yml passes STAGING_BASE_URL; promote.yml smoke-polls production without Playwright), so asserting the header there is safe. The two assertions live in one test because the halves only work together. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015mWQnpye9F299NVRCCSRvj --- e2e/tests/journeys.spec.ts | 31 +++++++++++++++++++++++++++++++ nextjs-app/next.config.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/e2e/tests/journeys.spec.ts b/e2e/tests/journeys.spec.ts index 2909136..1051e21 100644 --- a/e2e/tests/journeys.spec.ts +++ b/e2e/tests/journeys.spec.ts @@ -1600,3 +1600,34 @@ test('the sitemap submits no Welsh or overseas school', async ({ page }) => { expect(xml).not.toContain('/school/401559'); expect(xml).not.toContain('/school/402426'); }); + +/* + * Staging must not be indexable (spec 2026-08-20, W1 hygiene). + * + * These journeys only ever run against staging — deploy.yml passes + * STAGING_BASE_URL, and promote.yml only smoke-polls production without + * Playwright — so asserting the noindex header here is safe. + */ +test('staging answers noindex, and stays crawlable so the noindex is seen', async ({ page }) => { + const res = await page.request.get('/'); + expect(res.ok()).toBeTruthy(); + + const tag = res.headers()['x-robots-tag']; + expect(tag, 'staging must send X-Robots-Tag').toBeTruthy(); + expect(tag).toContain('noindex'); + + // The other half, and the reason this is one test rather than two: a + // Disallow would stop Google fetching the page at all, so it would never + // see the noindex above. The two only work together. + const robots = await (await page.request.get('/robots.txt')).text(); + expect(robots).not.toMatch(/^\s*Disallow:\s*\/\s*$/mi); +}); + +test('a school page on staging is noindexed too, not just the homepage', async ({ page }) => { + const list = await page.request.get('/api/schools?search=primary&per_page=1'); + const [first] = (await list.json()).schools ?? []; + expect(first, 'no school available').toBeTruthy(); + + const res = await page.request.get(`/school/${first.urn}-x`); + expect(res.headers()['x-robots-tag']).toContain('noindex'); +}); diff --git a/nextjs-app/next.config.js b/nextjs-app/next.config.js index 54474c0..c282b97 100644 --- a/nextjs-app/next.config.js +++ b/nextjs-app/next.config.js @@ -55,6 +55,37 @@ const nextConfig = { // Headers for caching and security async headers() { return [ + { + /* + * Keep non-production hosts out of the index. + * + * Staging serves the same image as production off stx., so without + * this it is a full crawlable duplicate of the site. + * + * X-Robots-Tag, NOT a robots.txt Disallow. Disallow blocks crawling, + * which is not the same as blocking indexing — a disallowed URL can + * still be indexed from external links, and worse, blocking the crawl + * means Google never fetches the page and never sees a noindex at all. + * Staging therefore stays crawlable and answers "noindex" when crawled. + * + * Matched on the staging host explicitly rather than "any host that is + * not production". The inverted form is tempting because it would cover + * future environments automatically, but its failure mode is + * deindexing production if the Host header ever arrives rewritten by a + * proxy. This form's failure mode is a new environment being indexable + * until someone adds it here — recoverable, where the other is not. + * + * Any new non-production hostname must be added to this list. + */ + source: '/:path*', + has: [{ type: 'host', value: 'stx.schoolcompare.co.uk' }], + headers: [ + { + key: 'X-Robots-Tag', + value: 'noindex, nofollow', + }, + ], + }, { source: '/:path*', headers: [ -- 2.54.0 From b34511e45980272267f720099086d9789a903f73 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 20 Aug 2026 23:23:30 +0100 Subject: [PATCH 2/2] chore: record the branch cleanup manifest 79 remote branches deleted: 77 fully merged into main, plus feat/seo-crawl-hygiene and feat/england-only-corpus, whose content is preserved on feat/seo-crawl-hygiene-main (PR #110). Each line carries the SHA, so any branch can be restored with git push origin :refs/heads/ The 14 branches left standing all carry content that differs from main and none of them is mine to judge abandoned. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015mWQnpye9F299NVRCCSRvj --- docs/branch-cleanup-2026-08-20.txt | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docs/branch-cleanup-2026-08-20.txt diff --git a/docs/branch-cleanup-2026-08-20.txt b/docs/branch-cleanup-2026-08-20.txt new file mode 100644 index 0000000..790dabd --- /dev/null +++ b/docs/branch-cleanup-2026-08-20.txt @@ -0,0 +1,85 @@ +# Remote branch cleanup, 2026-08-20 +# Restore any branch with: git push origin :refs/heads/ + +## Deleted: fully merged into main (content is in main) +75677f4252b759ef895e7d5f7c19f8f1745bdb59 add-contact-form-footer +fa1abff642683dfd26ba88a295a0a6710d77147a chore/byline-removal-and-audit-figure +95081d38bdf87764ef5d298676c25fae4cd3b792 chore/remove-parent-view +6877abedebfc1c7d95f1f6ebe68945c62be528ab chore/staged-prod-promotion +090d5f7bec824e083d3252e2c6e636686016304d ci/frontend-checks-speedup +8c3a5cc4e9f551f0190d85357ce7741ad87f3a4c design/cohort-identity +955659580067ca8b81bd77e31e4e2554103f094d feat/allow-analytics-iframe-embed +6828f6cd4417284ea3eb6f088fa20945b8b40ed3 feat/compare-chips-two-per-row +d5cd0abfee226885119665da5d2aa8288b59217f feat/compare-data-foundation +6dd9b04b50bee146682da87efad8fc8b526251c5 feat/compare-frontend-rebuild +96d5fcf5b07b6f175b48e9b20fcb765a320a907f feat/detail-header-details-reveal +f1388ff5bd0af1409823a1e047b7ba84246a0f70 feat/gias-sixth-form-flag +eddf74745f86c9c6d9eb07d867246ff9cc90dc20 feat/hero-artwork-v2 +3015c37bac6dc28db58c80fdb9235942c25b83d7 feat/hero-byline +8e763e39d17a4964cf558e51c03f044186371f6d feat/info-popover-tooltip +88c653215d520ab6e902c9de55bf27d86eeb90c3 feat/last-distance-offered +a72323874f7aebdb2e64b6d64a5febd61152d09d feat/last-distance-offered-full +c9a1892bfb0370e0672e5849cba294ddfabe3c65 feat/latest-cutoff-only +4e8df006d75d8be2a1d8529ddf855c445854cba1 feat/near-me-by-search +45ab479062c6a1639facad636fc0cc0cf0fd9155 feat/proposed-to-close-schools +3bf2e8f262cbe058fda6de6f8ea3e050224a51b9 feat/school-detail-visualisations +1f80571b1ff217dc92b660a936c02b5f49d07f0b feat/umami-heatmap-recorder +609bb923d96aa5730131463ea35d5efdd404bf96 feature/ingest-independent-schools +94151c58ea38a9256d66d15161293486a500c7a2 fix/admissions-section-height +79246edc22961c2beb3520437e9d064d3b809d10 fix/annual-dag-ks4-national-selector +59ac9c10b97e0ef1143f57fea06b324e72ac3d4a fix/chart-marker-contrast +9f8dba227c95706ca3527bd48d381e7622cc0a5e fix/compare-chart-refetch-resilience +e74d3882ce78a141fa1a57daa3102d7a58852dc3 fix/compare-expert-fixes +80176cac4db4820e76ea2a156c7a2974bee2f203 fix/compare-final-review-mustfix +f579630fab6c456e26a6984a3e8eebdbe3184202 fix/compare-mockup-drift +dc85254ad2ddf134b4434065d4762cef374d2220 fix/compare-null-year-blanks-chart +43a2c4a6bc539b621f31655aec05ef319a25f343 fix/compare-refresh-and-fetch +d677b5453365b72c81d6df2de62b1fa0d05d684d fix/daily-dag-cache-invalidation +f6bb037c471553e8195b5a8b147467ce0d07a688 fix/detail-all-through +17bd4d5a5eb0b12ca79b97db587f14f7d671e85b fix/detail-chart-truthfulness +4e6be0ce65647410b4ff74f8b763207c28920c26 fix/detail-inclusion-admissions +fdda52ff0af3fae03a4b059a655973cdbb269f91 fix/detail-ofsted-correctness +e36125b24aba254a8d15c5c33b4d2a296e691995 fix/detail-provenance-anchoring +b31e71ac884df6507f567adc046c9fc52d9d310a fix/detail-report-card-render-date +32f8a02862be6a1d49f4c3928b17fcf15d4c94cc fix/detail-trend-chart-taller +e65688d600a86818fe21ae4c61ba27e5b6ec8d7c fix/e2e-brand-assertions +3adea73ee04cdedfab54b0351878b297f72756ad fix/e2e-compare-chips-phase +06e4898c30feaedc471f97aba28ddb0d61379f4d fix/e2e-compare-samephase +9abd020967670a855e80fe5a908c8048a3aa9f14 fix/e2e-distance-locator +acec8135e1ec7c9c3c255e5b23733a0ef862b590 fix/e2e-rankings-year-pick +5944d88f0b1517ef1ef56af1b62270d2cf28e717 fix/expert-signoff-mustfixes +2433101fa08be5df6f170d41790512ffe823d33e fix/font-cascade-and-map-palette +74ca76d150deec6725259d9637ea86d7bb90c683 fix/gias-legacy-fallback +bdaa05cd542f563ef74c45307cd8f7fc465193c9 fix/hero-fallback-and-sharp +d52d384cf23d282b44e9251176f8f3402d808600 fix/hero-map-ios-fullscreen +4043270a77bbe4edb18207fa5f1d94d4747fe12f fix/hero-mobile-and-wording +22e9eb2d48b0d6623e88fd67cb6ca8e5e4074583 fix/homepage-education-accuracy +8d50afef1e8a2b621b7344eadf475b0d609ad7a2 fix/leaflet-specificity-and-font-assertion +b2b2cad5acf534ae7a667d3fb2be15efff37c4a7 fix/list-map-report-card-signal +dc21e80a5e9eebd13aaab84735642eb77cef35e6 fix/mobile-cell-name-size +e5f7f4c959f024c073472122d858333a0f24866c fix/mobile-compare-polish +a00cbe916182d1e04661c750a1ce2e7ad27a68ae fix/mobile-sort-select-overflow +2fd997bfe640c419a6713e85df008463de7f56e8 fix/modal-keyboard-viewport +3e7705756776a0c44d27966dfc972023f1f38b69 fix/ofsted-link-text +ce422e64363e2b03c186ef316832b6de15502d67 fix/promote-status-token +4522cbf64560db1e1cd519e119aa466b42cd16a1 fix/proposed-to-close-copy +15da060e4af37fbae919e0edf25f108266de2585 fix/rankings-admissions-accuracy +6c872ce726f210433354ca38dc5314bc6a467534 fix/rankings-year-validation +1c1df7796194af3d47f8e5ac0a0fbe6f323700e7 fix/report-card-chip-alignment +b2dc4d0779ced02709429afaa5985acf4abda794 fix/results-map-ios-fullscreen +95a5783da1fc994df76cb97238b55596dce4cd8f fix/runtime-api-proxy +8a9ba30cc24e29653a72c03c0e817684b7db7c07 fix/sats-per-level-national +536832a524fc4f9ed858f047e00b9a4d9d429c46 fix/school-detail-nan-500 +fef83b3bf244a9bf3cb4afa75dbc433d8725014f fix/schoolbar-sticky-offset +b0c5b6bb57c879477da12c37b15cff950c29ebd3 fix/secondary-anchors-button-affordance +261403bcd2b01aa4f26ee212e26302fc0f769bf9 fix/special-note-full-width +ea5249a2ea6faf6bfa5a1522644387ba59770ec8 fix/standardise-distance-units +e4565e9f158721d4df6b918f2b065de82851f8d9 fix/trends-chart-height +3aad5101a842539105022f9059e85c224fc5973a fix/welsh-establishment-leak +315f1feede70bdf3101d2fdd305b3d06e037fdac perf/batch-supplementary +d2dc78aeb599e16b7ef5019be2b360b08df463bc perf/compare-loading +e098ad4bd1130152705788d4773837b7d1e7112e perf/server-client-split + +## Deleted: superseded by PR #110 (content preserved on feat/seo-crawl-hygiene-main) +786ec80dd4de4a3cb674a89e35b3b5e461639289 feat/seo-crawl-hygiene +a5ac0bcd1b37bc10dcbce88f8601d01bf7b3eaaf feat/england-only-corpus -- 2.54.0