school_compare/nextjs-app/app/robots.ts

/**
 * Robots.txt Configuration
 * Controls search engine crawling behavior
 */

import { MetadataRoute } from 'next';
import { absoluteUrl } from '@/lib/site';

export default function robots(): MetadataRoute.Robots {
  return {
    rules: [
      {
        userAgent: '*',
        allow: '/',
        // /admin and /cms-api are also served X-Robots-Tag: noindex by
        // next.config.mjs. A Disallow alone blocks crawling, not indexing.
        disallow: ['/api/', '/_next/', '/admin/', '/cms-api/'],
      },
    ],
    sitemap: absoluteUrl('/sitemap.xml'),
  };
}