Merge pull request 'fix(seo): keep staging out of the search index' (#111) from fix/staging-noindex into main

Reviewed-on: #111
2026-08-20 22:39:38 +00:00
commit bb81337aba
@@ -0,0 +1,85 @@
+# Remote branch cleanup, 2026-08-20
+# Restore any branch with:  git push origin <sha>:refs/heads/<name>
+
+## Deleted: fully merged into main (content is in main)
+75677f4252b759ef895e7d5f7c19f8f1745bdb59 add-contact-form-footer
+fa1abff642683dfd26ba88a295a0a6710d77147a chore/byline-removal-and-audit-figure
+95081d38bdf87764ef5d298676c25fae4cd3b792 chore/remove-parent-view
+6877abedebfc1c7d95f1f6ebe68945c62be528ab chore/staged-prod-promotion
+090d5f7bec824e083d3252e2c6e636686016304d ci/frontend-checks-speedup
+8c3a5cc4e9f551f0190d85357ce7741ad87f3a4c design/cohort-identity
+955659580067ca8b81bd77e31e4e2554103f094d feat/allow-analytics-iframe-embed
+6828f6cd4417284ea3eb6f088fa20945b8b40ed3 feat/compare-chips-two-per-row
+d5cd0abfee226885119665da5d2aa8288b59217f feat/compare-data-foundation
+6dd9b04b50bee146682da87efad8fc8b526251c5 feat/compare-frontend-rebuild
+96d5fcf5b07b6f175b48e9b20fcb765a320a907f feat/detail-header-details-reveal
+f1388ff5bd0af1409823a1e047b7ba84246a0f70 feat/gias-sixth-form-flag
+eddf74745f86c9c6d9eb07d867246ff9cc90dc20 feat/hero-artwork-v2
+3015c37bac6dc28db58c80fdb9235942c25b83d7 feat/hero-byline
+8e763e39d17a4964cf558e51c03f044186371f6d feat/info-popover-tooltip
+88c653215d520ab6e902c9de55bf27d86eeb90c3 feat/last-distance-offered
+a72323874f7aebdb2e64b6d64a5febd61152d09d feat/last-distance-offered-full
+c9a1892bfb0370e0672e5849cba294ddfabe3c65 feat/latest-cutoff-only
+4e8df006d75d8be2a1d8529ddf855c445854cba1 feat/near-me-by-search
+45ab479062c6a1639facad636fc0cc0cf0fd9155 feat/proposed-to-close-schools
+3bf2e8f262cbe058fda6de6f8ea3e050224a51b9 feat/school-detail-visualisations
+1f80571b1ff217dc92b660a936c02b5f49d07f0b feat/umami-heatmap-recorder
+609bb923d96aa5730131463ea35d5efdd404bf96 feature/ingest-independent-schools
+94151c58ea38a9256d66d15161293486a500c7a2 fix/admissions-section-height
+79246edc22961c2beb3520437e9d064d3b809d10 fix/annual-dag-ks4-national-selector
+59ac9c10b97e0ef1143f57fea06b324e72ac3d4a fix/chart-marker-contrast
+9f8dba227c95706ca3527bd48d381e7622cc0a5e fix/compare-chart-refetch-resilience
+e74d3882ce78a141fa1a57daa3102d7a58852dc3 fix/compare-expert-fixes
+80176cac4db4820e76ea2a156c7a2974bee2f203 fix/compare-final-review-mustfix
+f579630fab6c456e26a6984a3e8eebdbe3184202 fix/compare-mockup-drift
+dc85254ad2ddf134b4434065d4762cef374d2220 fix/compare-null-year-blanks-chart
+43a2c4a6bc539b621f31655aec05ef319a25f343 fix/compare-refresh-and-fetch
+d677b5453365b72c81d6df2de62b1fa0d05d684d fix/daily-dag-cache-invalidation
+f6bb037c471553e8195b5a8b147467ce0d07a688 fix/detail-all-through
+17bd4d5a5eb0b12ca79b97db587f14f7d671e85b fix/detail-chart-truthfulness
+4e6be0ce65647410b4ff74f8b763207c28920c26 fix/detail-inclusion-admissions
+fdda52ff0af3fae03a4b059a655973cdbb269f91 fix/detail-ofsted-correctness
+e36125b24aba254a8d15c5c33b4d2a296e691995 fix/detail-provenance-anchoring
+b31e71ac884df6507f567adc046c9fc52d9d310a fix/detail-report-card-render-date
+32f8a02862be6a1d49f4c3928b17fcf15d4c94cc fix/detail-trend-chart-taller
+e65688d600a86818fe21ae4c61ba27e5b6ec8d7c fix/e2e-brand-assertions
+3adea73ee04cdedfab54b0351878b297f72756ad fix/e2e-compare-chips-phase
+06e4898c30feaedc471f97aba28ddb0d61379f4d fix/e2e-compare-samephase
+9abd020967670a855e80fe5a908c8048a3aa9f14 fix/e2e-distance-locator
+acec8135e1ec7c9c3c255e5b23733a0ef862b590 fix/e2e-rankings-year-pick
+5944d88f0b1517ef1ef56af1b62270d2cf28e717 fix/expert-signoff-mustfixes
+2433101fa08be5df6f170d41790512ffe823d33e fix/font-cascade-and-map-palette
+74ca76d150deec6725259d9637ea86d7bb90c683 fix/gias-legacy-fallback
+bdaa05cd542f563ef74c45307cd8f7fc465193c9 fix/hero-fallback-and-sharp
+d52d384cf23d282b44e9251176f8f3402d808600 fix/hero-map-ios-fullscreen
+4043270a77bbe4edb18207fa5f1d94d4747fe12f fix/hero-mobile-and-wording
+22e9eb2d48b0d6623e88fd67cb6ca8e5e4074583 fix/homepage-education-accuracy
+8d50afef1e8a2b621b7344eadf475b0d609ad7a2 fix/leaflet-specificity-and-font-assertion
+b2b2cad5acf534ae7a667d3fb2be15efff37c4a7 fix/list-map-report-card-signal
+dc21e80a5e9eebd13aaab84735642eb77cef35e6 fix/mobile-cell-name-size
+e5f7f4c959f024c073472122d858333a0f24866c fix/mobile-compare-polish
+a00cbe916182d1e04661c750a1ce2e7ad27a68ae fix/mobile-sort-select-overflow
+2fd997bfe640c419a6713e85df008463de7f56e8 fix/modal-keyboard-viewport
+3e7705756776a0c44d27966dfc972023f1f38b69 fix/ofsted-link-text
+ce422e64363e2b03c186ef316832b6de15502d67 fix/promote-status-token
+4522cbf64560db1e1cd519e119aa466b42cd16a1 fix/proposed-to-close-copy
+15da060e4af37fbae919e0edf25f108266de2585 fix/rankings-admissions-accuracy
+6c872ce726f210433354ca38dc5314bc6a467534 fix/rankings-year-validation
+1c1df7796194af3d47f8e5ac0a0fbe6f323700e7 fix/report-card-chip-alignment
+b2dc4d0779ced02709429afaa5985acf4abda794 fix/results-map-ios-fullscreen
+95a5783da1fc994df76cb97238b55596dce4cd8f fix/runtime-api-proxy
+8a9ba30cc24e29653a72c03c0e817684b7db7c07 fix/sats-per-level-national
+536832a524fc4f9ed858f047e00b9a4d9d429c46 fix/school-detail-nan-500
+fef83b3bf244a9bf3cb4afa75dbc433d8725014f fix/schoolbar-sticky-offset
+b0c5b6bb57c879477da12c37b15cff950c29ebd3 fix/secondary-anchors-button-affordance
+261403bcd2b01aa4f26ee212e26302fc0f769bf9 fix/special-note-full-width
+ea5249a2ea6faf6bfa5a1522644387ba59770ec8 fix/standardise-distance-units
+e4565e9f158721d4df6b918f2b065de82851f8d9 fix/trends-chart-height
+3aad5101a842539105022f9059e85c224fc5973a fix/welsh-establishment-leak
+315f1feede70bdf3101d2fdd305b3d06e037fdac perf/batch-supplementary
+d2dc78aeb599e16b7ef5019be2b360b08df463bc perf/compare-loading
+e098ad4bd1130152705788d4773837b7d1e7112e perf/server-client-split
+
+## Deleted: superseded by PR #110 (content preserved on feat/seo-crawl-hygiene-main)
+786ec80dd4de4a3cb674a89e35b3b5e461639289 feat/seo-crawl-hygiene
+a5ac0bcd1b37bc10dcbce88f8601d01bf7b3eaaf feat/england-only-corpus
@@ -1695,3 +1695,34 @@ test('a bare /compare is indexable, a parameterised one is not', async ({ page }
    .getAttribute('href');
  expect(canonical).toBe('https://www.schoolcompare.co.uk/compare');
 });
+
+/*
+ * Staging must not be indexable (spec 2026-08-20, W1 hygiene).
+ *
+ * These journeys only ever run against staging — deploy.yml passes
+ * STAGING_BASE_URL, and promote.yml only smoke-polls production without
+ * Playwright — so asserting the noindex header here is safe.
+ */
+test('staging answers noindex, and stays crawlable so the noindex is seen', async ({ page }) => {
+  const res = await page.request.get('/');
+  expect(res.ok()).toBeTruthy();
+
+  const tag = res.headers()['x-robots-tag'];
+  expect(tag, 'staging must send X-Robots-Tag').toBeTruthy();
+  expect(tag).toContain('noindex');
+
+  // The other half, and the reason this is one test rather than two: a
+  // Disallow would stop Google fetching the page at all, so it would never
+  // see the noindex above. The two only work together.
+  const robots = await (await page.request.get('/robots.txt')).text();
+  expect(robots).not.toMatch(/^\s*Disallow:\s*\/\s*$/mi);
+});
+
+test('a school page on staging is noindexed too, not just the homepage', async ({ page }) => {
+  const list = await page.request.get('/api/schools?search=primary&per_page=1');
+  const [first] = (await list.json()).schools ?? [];
+  expect(first, 'no school available').toBeTruthy();
+
+  const res = await page.request.get(`/school/${first.urn}-x`);
+  expect(res.headers()['x-robots-tag']).toContain('noindex');
+});
@@ -55,6 +55,37 @@ const nextConfig = {
  // Headers for caching and security
  async headers() {
    return [
+      {
+        /*
+         * Keep non-production hosts out of the index.
+         *
+         * Staging serves the same image as production off stx., so without
+         * this it is a full crawlable duplicate of the site.
+         *
+         * X-Robots-Tag, NOT a robots.txt Disallow. Disallow blocks crawling,
+         * which is not the same as blocking indexing — a disallowed URL can
+         * still be indexed from external links, and worse, blocking the crawl
+         * means Google never fetches the page and never sees a noindex at all.
+         * Staging therefore stays crawlable and answers "noindex" when crawled.
+         *
+         * Matched on the staging host explicitly rather than "any host that is
+         * not production". The inverted form is tempting because it would cover
+         * future environments automatically, but its failure mode is
+         * deindexing production if the Host header ever arrives rewritten by a
+         * proxy. This form's failure mode is a new environment being indexable
+         * until someone adds it here — recoverable, where the other is not.
+         *
+         * Any new non-production hostname must be added to this list.
+         */
+        source: '/:path*',
+        has: [{ type: 'host', value: 'stx.schoolcompare.co.uk' }],
+        headers: [
+          {
+            key: 'X-Robots-Tag',
+            value: 'noindex, nofollow',
+          },
+        ],
+      },
      {
        source: '/:path*',
        headers: [