feat(seo): static sitemap generation job via Airflow
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 45s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m5s
Build and Push Docker Images / Build Pipeline (Meltano + dbt + Airflow) (push) Successful in 1m29s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 0s

- Backend builds sitemap.xml from school data at startup (in-memory)
- POST /api/admin/regenerate-sitemap refreshes it after data updates
- New Airflow DAG (sitemap_generate) runs Sundays 05:00 and calls the endpoint
- Next.js proxies /sitemap.xml to the backend; removes the slow dynamic sitemap.ts
- docker-compose passes BACKEND_URL + ADMIN_API_KEY to Airflow env

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-29 15:15:41 +01:00
parent 748891ab31
commit b7bff7bf6b
5 changed files with 156 additions and 56 deletions

View File

@@ -1,55 +0,0 @@
/**
* Dynamic Sitemap Generation
* Generates sitemap with all school pages and main routes
*/
import { MetadataRoute } from 'next';
import { fetchSchools } from '@/lib/api';
import { schoolUrl } from '@/lib/utils';
const BASE_URL = 'https://schoolcompare.co.uk';
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
// Static pages
const staticPages: MetadataRoute.Sitemap = [
{
url: BASE_URL,
lastModified: new Date(),
changeFrequency: 'daily',
priority: 1.0,
},
{
url: `${BASE_URL}/compare`,
lastModified: new Date(),
changeFrequency: 'weekly',
priority: 0.8,
},
{
url: `${BASE_URL}/rankings`,
lastModified: new Date(),
changeFrequency: 'weekly',
priority: 0.8,
},
];
// Fetch all schools (in batches if necessary)
try {
const schoolsData = await fetchSchools({
page: 1,
page_size: 10000, // Fetch all schools
});
const schoolPages: MetadataRoute.Sitemap = schoolsData.schools.map((school) => ({
url: `${BASE_URL}${schoolUrl(school.urn, school.school_name)}`,
lastModified: new Date(),
changeFrequency: 'monthly',
priority: 0.6,
}));
return [...staticPages, ...schoolPages];
} catch (error) {
console.error('Failed to generate sitemap:', error);
// Return just static pages if school fetch fails
return staticPages;
}
}