TudorandClaude Opus 5
c2c76c5817
X-Robots-Tag rather than the robots.txt Disallow alone, for the same reason the staging rule uses one: a Disallow blocks crawling, not indexing, so a URL found from an external link can be indexed without ever being fetched — and blocking the crawl means the noindex is never seen. Both mechanisms are applied to /admin and /cms-api. The existing CSP is frame-ancestors only, which restricts who may embed the site rather than what a page may load, so it cannot break the panel. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017YmbBhr8s7GusjDE12hrZM
23 lines
572 B
TypeScript
23 lines
572 B
TypeScript
/**
|
|
* Robots.txt Configuration
|
|
* Controls search engine crawling behavior
|
|
*/
|
|
|
|
import { MetadataRoute } from 'next';
|
|
import { absoluteUrl } from '@/lib/site';
|
|
|
|
export default function robots(): MetadataRoute.Robots {
|
|
return {
|
|
rules: [
|
|
{
|
|
userAgent: '*',
|
|
allow: '/',
|
|
// /admin and /cms-api are also served X-Robots-Tag: noindex by
|
|
// next.config.mjs. A Disallow alone blocks crawling, not indexing.
|
|
disallow: ['/api/', '/_next/', '/admin/', '/cms-api/'],
|
|
},
|
|
],
|
|
sitemap: absoluteUrl('/sitemap.xml'),
|
|
};
|
|
}
|