fix(ks2): make reimport async with polling to avoid HTTP timeout
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 47s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m12s
Build and Push Docker Images / Build Integrator (push) Successful in 58s
Build and Push Docker Images / Build Kestra Init (push) Successful in 31s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 1s

The geocoding pass over ~15k schools takes longer than any reasonable
HTTP timeout. New approach:
- POST /api/admin/reimport-ks2 starts migration in background thread,
  returns {"status":"started"} immediately
- GET /api/admin/reimport-ks2/status returns {running, done}
- ks2.py polls status every 30s (max 2h) before returning
- Kestra flow timeout bumped to PT2H

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-24 21:08:06 +00:00
parent 68b15400b0
commit 0e5b71d4a0
3 changed files with 71 additions and 23 deletions

View File

@@ -635,6 +635,9 @@ async def reload_data(
return {"status": "reloaded"}
_reimport_status: dict = {"running": False, "done": False, "error": None}
@app.post("/api/admin/reimport-ks2")
@limiter.limit("2/minute")
async def reimport_ks2(
@@ -643,19 +646,45 @@ async def reimport_ks2(
_: bool = Depends(verify_admin_api_key)
):
"""
Re-run the full KS2 CSV migration (drop + reimport schools and results).
Use when the database has been wiped and needs repopulating from the CSV files.
Start a full KS2 CSV migration in the background and return immediately.
Poll GET /api/admin/reimport-ks2/status to check progress.
Pass ?geocode=false to skip postcode → lat/lng resolution.
Runs synchronously — expect this to take several minutes.
Requires X-API-Key header with valid admin API key.
"""
loop = asyncio.get_event_loop()
success = await loop.run_in_executor(None, lambda: run_full_migration(geocode=geocode))
if not success:
raise HTTPException(status_code=500, detail="Migration failed: no CSV data found in data directory")
clear_cache()
load_school_data()
return {"status": "reimported"}
global _reimport_status
if _reimport_status["running"]:
return {"status": "already_running"}
_reimport_status = {"running": True, "done": False, "error": None}
def _run():
global _reimport_status
try:
success = run_full_migration(geocode=geocode)
if not success:
_reimport_status = {"running": False, "done": False, "error": "No CSV data found"}
return
clear_cache()
load_school_data()
_reimport_status = {"running": False, "done": True, "error": None}
except Exception as exc:
_reimport_status = {"running": False, "done": False, "error": str(exc)}
import threading
threading.Thread(target=_run, daemon=True).start()
return {"status": "started"}
@app.get("/api/admin/reimport-ks2/status")
async def reimport_ks2_status(
request: Request,
_: bool = Depends(verify_admin_api_key)
):
"""Poll this endpoint to check reimport progress."""
s = _reimport_status
if s["error"]:
raise HTTPException(status_code=500, detail=s["error"])
return {"running": s["running"], "done": s["done"]}
# =============================================================================