"""Ofsted MI Singer tap — extracts inspection records from GOV.UK CSV/ODS."""

from __future__ import annotations

from datetime import datetime
import io
import re

import requests
from singer_sdk import Stream, Tap
from singer_sdk import typing as th

GOV_UK_PAGE = (
    "https://www.gov.uk/government/statistical-data-sets/"
    "monthly-management-information-ofsteds-school-inspections-outcomes"
)

INDEPENDENT_GOV_UK_PAGE = (
    "https://www.gov.uk/government/statistical-data-sets/"
    "non-association-independent-schools-inspections-and-outcomes-management-information"
)

# Column name → internal field, in priority order (first match wins).
# Handles both current and older file formats.
COLUMN_PRIORITY = {
    "urn": ["URN", "Urn", "urn"],
    "inspection_date": [
        "Inspection start date of latest OEIF graded inspection",
        "Inspection start date of latest OEIF standard inspection",
        "Inspection start date",
        "Inspection date",
    ],
    "inspection_type": [
        "Inspection type of latest OEIF graded inspection",
        "Inspection type of latest OEIF standard inspection",
        "Inspection type",
    ],
    "event_type_grouping": [
        "Event type grouping of latest OEIF standard inspection",
        "Event type grouping",
        "Inspection type grouping",
    ],
    "overall_effectiveness": [
        "Latest OEIF overall effectiveness",
        "Overall effectiveness",
    ],
    "quality_of_education": [
        "Latest OEIF quality of education",
        "Quality of education",
    ],
    "behaviour_and_attitudes": [
        "Latest OEIF behaviour and attitudes",
        "Behaviour and attitudes",
    ],
    "personal_development": [
        "Latest OEIF personal development",
        "Personal development",
    ],
    "effectiveness_of_leadership_and_management": [
        "Latest OEIF effectiveness of leadership and management",
        "Effectiveness of leadership and management",
    ],
    "early_years_provision": [
        "Latest OEIF early years provision (where applicable)",
        "Latest OEIF early years provision",
        "Early years provision (where applicable)",
    ],
    "sixth_form_provision": [
        "Latest OEIF sixth form provision (where applicable)",
        "Latest OEIF sixth form provision",
        "Sixth form provision (where applicable)",
    ],
    # Ungraded (Section 8) inspection — no fresh grade is assigned; the outcome
    # is free text like "School remains Good". Used as a last-resort fallback
    # for the school's grade when no graded overall effectiveness is available.
    "ungraded_outcome": [
        "Ungraded inspection overall outcome",
    ],
    "ungraded_inspection_date": [
        "Date of latest ungraded inspection",
    ],
    # Report Card fields (post-Nov 2025 framework).
    "rc_safeguarding_met": ["Safeguarding standards"],
    "rc_inclusion": ["Inclusion"],
    "rc_curriculum_teaching": ["Curriculum and teaching"],
    "rc_achievement": ["Achievement"],
    "rc_attendance_behaviour": ["Attendance and behaviour"],
    "rc_personal_development": ["Personal development and wellbeing"],
    "rc_leadership_governance": ["Leadership and governance"],
    "rc_early_years": ["Early years (where applicable)"],
    "rc_sixth_form": ["Post-16 provision (where applicable)"],
    # Date of the latest FULL inspection — in the renewed framework this is
    # the report-card inspection's own start date (col "Inspection start
    # date"), distinct from the legacy OEIF graded/ungraded dates above.
    "rc_inspection_date": ["Inspection start date"],
    "report_url": [
        "Web Link (opens in new window)",
        "Web link to Ofsted provider page",
        "Web link",
    ],
}


def discover_csv_url() -> str | None:
    """Scrape GOV.UK page to find the latest MI CSV download link.

    The page lists a decade of monthly files, oldest first — take the
    newest 'latest inspections as at <date>' link by parsing its date,
    never matches[0] (that is a 2017 file).
    """
    resp = requests.get(GOV_UK_PAGE, timeout=30)
    resp.raise_for_status()
    csv_links = re.findall(
        r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.csv)"',
        resp.text,
    )

    months = {
        'january': 1, 'february': 2, 'march': 3, 'april': 4, 'may': 5, 'june': 6,
        'july': 7, 'august': 8, 'september': 9, 'october': 10, 'november': 11, 'december': 12,
        'jan': 1, 'feb': 2, 'mar': 3, 'apr': 4, 'jun': 6,
        'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12,
    }
    parsed_links = []
    for link in csv_links:
        normalized = link.lower().replace('-', '_')
        if 'latest_inspections_as_at' not in normalized:
            continue
        match = re.search(r'as_at_(\d{1,2})_([a-z]+)_(\d{4})', normalized)
        if match:
            day, month_str, year = match.groups()
            month = months.get(month_str)
            if month:
                try:
                    parsed_links.append((datetime(int(year), month, int(day)), link))
                except ValueError:
                    continue
    parsed_links.sort(reverse=True)
    if parsed_links:
        return parsed_links[0][1]
    if csv_links:
        return csv_links[-1]
    # Fall back to ODS
    matches = re.findall(
        r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.ods)"',
        resp.text,
    )
    return matches[0] if matches else None


def discover_independent_csv_url() -> str | None:
    """Scrape GOV.UK page to find the latest independent schools MI CSV download link."""
    resp = requests.get(INDEPENDENT_GOV_UK_PAGE, timeout=30)
    resp.raise_for_status()
    # Look for CSV attachment links
    csv_links = re.findall(
        r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.csv)"',
        resp.text,
    )
    if not csv_links:
        # Fall back to ODS
        csv_links = re.findall(
            r'href="(https://assets\.publishing\.service\.gov\.uk/[^"]+\.ods)"',
            resp.text,
        )

    months = {
        'january': 1, 'february': 2, 'march': 3, 'april': 4, 'may': 5, 'june': 6,
        'july': 7, 'august': 8, 'september': 9, 'october': 10, 'november': 11, 'december': 12
    }

    parsed_links = []
    for link in csv_links:
        normalized_link = link.lower().replace('-', '_')
        if 'most_recent' not in normalized_link:
            continue

        match = re.search(r'as_at_(\d{1,2})_([a-z]+)_(\d{4})', normalized_link)
        if match:
            day, month_str, year = match.groups()
            month = months.get(month_str)
            if month:
                try:
                    dt = datetime(int(year), month, int(day))
                    parsed_links.append((dt, link))
                except ValueError:
                    continue

    parsed_links.sort(reverse=True)
    if parsed_links:
        return parsed_links[0][1]

    return csv_links[0] if csv_links else None


class OfstedInspectionsStream(Stream):
    """Stream: Ofsted inspection records."""

    name = "ofsted_inspections"
    primary_keys = ["urn", "inspection_date"]
    replication_key = None

    schema = th.PropertiesList(
        th.Property("urn", th.IntegerType, required=True),
        th.Property("inspection_date", th.StringType),
        th.Property("inspection_type", th.StringType),
        th.Property("event_type_grouping", th.StringType),
        th.Property("overall_effectiveness", th.StringType),
        th.Property("quality_of_education", th.StringType),
        th.Property("behaviour_and_attitudes", th.StringType),
        th.Property("personal_development", th.StringType),
        th.Property("effectiveness_of_leadership_and_management", th.StringType),
        th.Property("early_years_provision", th.StringType),
        th.Property("sixth_form_provision", th.StringType),
        th.Property("ungraded_outcome", th.StringType),
        th.Property("ungraded_inspection_date", th.StringType),
        th.Property("rc_safeguarding_met", th.StringType),
        th.Property("rc_inclusion", th.StringType),
        th.Property("rc_curriculum_teaching", th.StringType),
        th.Property("rc_achievement", th.StringType),
        th.Property("rc_attendance_behaviour", th.StringType),
        th.Property("rc_personal_development", th.StringType),
        th.Property("rc_leadership_governance", th.StringType),
        th.Property("rc_early_years", th.StringType),
        th.Property("rc_sixth_form", th.StringType),
        th.Property("rc_inspection_date", th.StringType),
        th.Property("report_url", th.StringType),
    ).to_dict()

    def _resolve_columns(self, df_columns: list[str]) -> dict[str, str]:
        """Map internal field names to actual CSV column names."""
        mapping = {}
        for field, candidates in COLUMN_PRIORITY.items():
            for candidate in candidates:
                if candidate in df_columns:
                    mapping[field] = candidate
                    break
        return mapping

    def _fetch_and_parse_url(self, url: str, pd) -> list[dict]:
        """Download file and parse records."""
        resp = requests.get(url, timeout=120)
        resp.raise_for_status()

        if url.endswith(".ods"):
            df = pd.read_excel(io.BytesIO(resp.content), engine="odf", dtype=str)
        else:
            # Detect header row (may not be row 0)
            # Try utf-8-sig first, fall back to latin-1 (matches integrator)
            try:
                text = resp.content.decode("utf-8-sig")
            except UnicodeDecodeError:
                text = resp.content.decode("latin-1")
            lines = text.split("\n")
            header_idx = 0
            for i, line in enumerate(lines[:20]):
                if re.search(r'(?:^|,)\s*URN\s*(?:,|$)', line):
                    header_idx = i
                    break
            df = pd.read_csv(
                io.StringIO(text),
                skiprows=header_idx,
                dtype=str,
                keep_default_na=False,
            )

        col_map = self._resolve_columns(list(df.columns))
        self.logger.info("CSV columns: %s", list(df.columns))
        self.logger.info("Resolved column map: %s", col_map)

        for _, row in df.iterrows():
            record = {}
            for field, col in col_map.items():
                val = row.get(col, None)
                if pd.isna(val):
                    val = None
                record[field] = val

            # Cast URN
            try:
                record["urn"] = int(record.get("urn"))
            except (ValueError, KeyError, TypeError):
                continue

            yield record

    def get_records(self, context):
        import pandas as pd

        # 1. State-funded schools
        state_url = self.config.get("mi_url") or discover_csv_url()
        if state_url:
            self.logger.info("Downloading Ofsted state-funded MI: %s", state_url)
            yield from self._fetch_and_parse_url(state_url, pd)
        else:
            self.logger.error("Could not discover Ofsted state-funded MI download URL")

        # 2. Independent schools
        ind_url = self.config.get("independent_mi_url") or discover_independent_csv_url()
        if ind_url:
            self.logger.info("Downloading Ofsted independent MI: %s", ind_url)
            yield from self._fetch_and_parse_url(ind_url, pd)
        else:
            self.logger.error("Could not discover Ofsted independent MI download URL")


class TapUKOfsted(Tap):
    """Singer tap for UK Ofsted Management Information."""

    name = "tap-uk-ofsted"

    config_jsonschema = th.PropertiesList(
        th.Property("mi_url", th.StringType, description="Direct URL to Ofsted MI file"),
        th.Property("independent_mi_url", th.StringType, description="Direct URL to Ofsted Independent Schools MI file"),
    ).to_dict()

    def discover_streams(self):
        return [OfstedInspectionsStream(self)]


if __name__ == "__main__":
    TapUKOfsted.cli()
