fix(ks2): make reimport async with polling to avoid HTTP timeout
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 47s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m12s
Build and Push Docker Images / Build Integrator (push) Successful in 58s
Build and Push Docker Images / Build Kestra Init (push) Successful in 31s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 1s
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 47s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m12s
Build and Push Docker Images / Build Integrator (push) Successful in 58s
Build and Push Docker Images / Build Kestra Init (push) Successful in 31s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 1s
The geocoding pass over ~15k schools takes longer than any reasonable
HTTP timeout. New approach:
- POST /api/admin/reimport-ks2 starts migration in background thread,
returns {"status":"started"} immediately
- GET /api/admin/reimport-ks2/status returns {running, done}
- ks2.py polls status every 30s (max 2h) before returning
- Kestra flow timeout bumped to PT2H
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -635,6 +635,9 @@ async def reload_data(
|
||||
return {"status": "reloaded"}
|
||||
|
||||
|
||||
_reimport_status: dict = {"running": False, "done": False, "error": None}
|
||||
|
||||
|
||||
@app.post("/api/admin/reimport-ks2")
|
||||
@limiter.limit("2/minute")
|
||||
async def reimport_ks2(
|
||||
@@ -643,19 +646,45 @@ async def reimport_ks2(
|
||||
_: bool = Depends(verify_admin_api_key)
|
||||
):
|
||||
"""
|
||||
Re-run the full KS2 CSV migration (drop + reimport schools and results).
|
||||
Use when the database has been wiped and needs repopulating from the CSV files.
|
||||
Start a full KS2 CSV migration in the background and return immediately.
|
||||
Poll GET /api/admin/reimport-ks2/status to check progress.
|
||||
Pass ?geocode=false to skip postcode → lat/lng resolution.
|
||||
Runs synchronously — expect this to take several minutes.
|
||||
Requires X-API-Key header with valid admin API key.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
success = await loop.run_in_executor(None, lambda: run_full_migration(geocode=geocode))
|
||||
if not success:
|
||||
raise HTTPException(status_code=500, detail="Migration failed: no CSV data found in data directory")
|
||||
clear_cache()
|
||||
load_school_data()
|
||||
return {"status": "reimported"}
|
||||
global _reimport_status
|
||||
if _reimport_status["running"]:
|
||||
return {"status": "already_running"}
|
||||
|
||||
_reimport_status = {"running": True, "done": False, "error": None}
|
||||
|
||||
def _run():
|
||||
global _reimport_status
|
||||
try:
|
||||
success = run_full_migration(geocode=geocode)
|
||||
if not success:
|
||||
_reimport_status = {"running": False, "done": False, "error": "No CSV data found"}
|
||||
return
|
||||
clear_cache()
|
||||
load_school_data()
|
||||
_reimport_status = {"running": False, "done": True, "error": None}
|
||||
except Exception as exc:
|
||||
_reimport_status = {"running": False, "done": False, "error": str(exc)}
|
||||
|
||||
import threading
|
||||
threading.Thread(target=_run, daemon=True).start()
|
||||
return {"status": "started"}
|
||||
|
||||
|
||||
@app.get("/api/admin/reimport-ks2/status")
|
||||
async def reimport_ks2_status(
|
||||
request: Request,
|
||||
_: bool = Depends(verify_admin_api_key)
|
||||
):
|
||||
"""Poll this endpoint to check reimport progress."""
|
||||
s = _reimport_status
|
||||
if s["error"]:
|
||||
raise HTTPException(status_code=500, detail=s["error"])
|
||||
return {"running": s["running"], "done": s["done"]}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user