school_compare/nextjs-app/app/robots.ts

/**
 * Robots.txt Configuration
 * Controls search engine crawling behavior
 */

import { MetadataRoute } from 'next';
import { absoluteUrl } from '@/lib/site';

export default function robots(): MetadataRoute.Robots {
  return {
    rules: [
      {
        userAgent: '*',
        allow: '/',
        // /admin and /cms-api are also served X-Robots-Tag: noindex by
        // next.config.mjs. A Disallow alone blocks crawling, not indexing.
        disallow: ['/api/', '/_next/', '/admin/', '/cms-api/'],
      },
    ],
    // Two sitemaps: /sitemap.xml is proxied from FastAPI and carries the
    // school corpus; /content-sitemap.xml is Next-owned and carries /about
    // and the blog. The backend knows nothing about Payload.
    sitemap: [absoluteUrl('/sitemap.xml'), absoluteUrl('/content-sitemap.xml')],
  };
}