school_compare/backend/schemas.py

"""
Schema definitions: column mappings, metric definitions, school type mappings.
Single source of truth for all data transformations.
"""

# Column name mappings from DfE CSV to API field names
COLUMN_MAPPINGS = {
    'URN': 'urn',
    'SCHNAME': 'school_name',
    'ADDRESS1': 'address1',
    'ADDRESS2': 'address2',
    'TOWN': 'town',
    'PCODE': 'postcode',
    'NFTYPE': 'school_type_code',
    'RELDENOM': 'religious_denomination',
    'AGERANGE': 'age_range',
    'TOTPUPS': 'total_pupils',
    'TELIG': 'eligible_pupils',
    # Core KS2 metrics
    'PTRWM_EXP': 'rwm_expected_pct',
    'PTRWM_HIGH': 'rwm_high_pct',
    'READPROG': 'reading_progress',
    'WRITPROG': 'writing_progress',
    'MATPROG': 'maths_progress',
    'PTREAD_EXP': 'reading_expected_pct',
    'PTWRITTA_EXP': 'writing_expected_pct',
    'PTMAT_EXP': 'maths_expected_pct',
    'READ_AVERAGE': 'reading_avg_score',
    'MAT_AVERAGE': 'maths_avg_score',
    'PTREAD_HIGH': 'reading_high_pct',
    'PTWRITTA_HIGH': 'writing_high_pct',
    'PTMAT_HIGH': 'maths_high_pct',
    # GPS (Grammar, Punctuation & Spelling)
    'PTGPS_EXP': 'gps_expected_pct',
    'PTGPS_HIGH': 'gps_high_pct',
    'GPS_AVERAGE': 'gps_avg_score',
    # Science
    'PTSCITA_EXP': 'science_expected_pct',
    # School context
    'PTFSM6CLA1A': 'disadvantaged_pct',
    'PTEALGRP2': 'eal_pct',
    'PSENELK': 'sen_support_pct',
    'PSENELE': 'sen_ehcp_pct',
    'PTMOBN': 'stability_pct',
    # Gender breakdown
    'PTRWM_EXP_B': 'rwm_expected_boys_pct',
    'PTRWM_EXP_G': 'rwm_expected_girls_pct',
    'PTRWM_HIGH_B': 'rwm_high_boys_pct',
    'PTRWM_HIGH_G': 'rwm_high_girls_pct',
    # Disadvantaged performance
    'PTRWM_EXP_FSM6CLA1A': 'rwm_expected_disadvantaged_pct',
    'PTRWM_EXP_NotFSM6CLA1A': 'rwm_expected_non_disadvantaged_pct',
    'DIFFN_RWM_EXP': 'disadvantaged_gap',
    # 3-year averages
    'PTRWM_EXP_3YR': 'rwm_expected_3yr_pct',
    'READ_AVERAGE_3YR': 'reading_avg_3yr',
    'MAT_AVERAGE_3YR': 'maths_avg_3yr',
}

# Numeric columns that need parsing
NUMERIC_COLUMNS = [
    # Core metrics
    'rwm_expected_pct', 'rwm_high_pct', 'reading_progress', 
    'writing_progress', 'maths_progress', 'reading_expected_pct',
    'writing_expected_pct', 'maths_expected_pct', 'reading_avg_score',
    'maths_avg_score', 'reading_high_pct', 'writing_high_pct', 'maths_high_pct',
    # GPS & Science
    'gps_expected_pct', 'gps_high_pct', 'gps_avg_score', 'science_expected_pct',
    # School context
    'total_pupils', 'eligible_pupils', 'disadvantaged_pct', 'eal_pct',
    'sen_support_pct', 'sen_ehcp_pct', 'stability_pct',
    # Gender breakdown
    'rwm_expected_boys_pct', 'rwm_expected_girls_pct',
    'rwm_high_boys_pct', 'rwm_high_girls_pct',
    # Disadvantaged performance
    'rwm_expected_disadvantaged_pct', 'rwm_expected_non_disadvantaged_pct', 'disadvantaged_gap',
    # 3-year averages
    'rwm_expected_3yr_pct', 'reading_avg_3yr', 'maths_avg_3yr',
]

# School type code to name mapping
SCHOOL_TYPE_MAP = {
    'AC': 'Academy',
    'ACC': 'Academy Converter',
    'ACS': 'Academy Sponsor Led',
    'CY': 'Community School',
    'VA': 'Voluntary Aided',
    'VC': 'Voluntary Controlled',
    'FD': 'Foundation',
    'F': 'Foundation',
    'FS': 'Free School',
}

# Special values to treat as null
NULL_VALUES = ['SUPP', 'NE', 'NA', 'NP', 'NEW', 'LOW', '']

# KS2 Metric definitions - single source of truth
# Used by both backend API and frontend
METRIC_DEFINITIONS = {
    # Expected Standard
    "rwm_expected_pct": {
        "name": "RWM Combined %",
        "short_name": "RWM %",
        "description": "% meeting expected standard in reading, writing and maths",
        "type": "percentage",
        "category": "expected"
    },
    "reading_expected_pct": {
        "name": "Reading Expected %",
        "short_name": "Reading %",
        "description": "% meeting expected standard in reading",
        "type": "percentage",
        "category": "expected"
    },
    "writing_expected_pct": {
        "name": "Writing Expected %",
        "short_name": "Writing %",
        "description": "% meeting expected standard in writing",
        "type": "percentage",
        "category": "expected"
    },
    "maths_expected_pct": {
        "name": "Maths Expected %",
        "short_name": "Maths %",
        "description": "% meeting expected standard in maths",
        "type": "percentage",
        "category": "expected"
    },
    "gps_expected_pct": {
        "name": "GPS Expected %",
        "short_name": "GPS %",
        "description": "% meeting expected standard in grammar, punctuation & spelling",
        "type": "percentage",
        "category": "expected"
    },
    "science_expected_pct": {
        "name": "Science Expected %",
        "short_name": "Science %",
        "description": "% meeting expected standard in science",
        "type": "percentage",
        "category": "expected"
    },
    # Higher Standard
    "rwm_high_pct": {
        "name": "RWM Combined Higher %",
        "short_name": "RWM Higher %",
        "description": "% achieving higher standard in RWM combined",
        "type": "percentage",
        "category": "higher"
    },
    "reading_high_pct": {
        "name": "Reading Higher %",
        "short_name": "Reading Higher %",
        "description": "% achieving higher standard in reading",
        "type": "percentage",
        "category": "higher"
    },
    "writing_high_pct": {
        "name": "Writing Higher %",
        "short_name": "Writing Higher %",
        "description": "% achieving greater depth in writing",
        "type": "percentage",
        "category": "higher"
    },
    "maths_high_pct": {
        "name": "Maths Higher %",
        "short_name": "Maths Higher %",
        "description": "% achieving higher standard in maths",
        "type": "percentage",
        "category": "higher"
    },
    "gps_high_pct": {
        "name": "GPS Higher %",
        "short_name": "GPS Higher %",
        "description": "% achieving higher standard in GPS",
        "type": "percentage",
        "category": "higher"
    },
    # Progress Scores
    "reading_progress": {
        "name": "Reading Progress",
        "short_name": "Reading Progress",
        "description": "Progress in reading from KS1 to KS2",
        "type": "score",
        "category": "progress"
    },
    "writing_progress": {
        "name": "Writing Progress",
        "short_name": "Writing Progress",
        "description": "Progress in writing from KS1 to KS2",
        "type": "score",
        "category": "progress"
    },
    "maths_progress": {
        "name": "Maths Progress",
        "short_name": "Maths Progress",
        "description": "Progress in maths from KS1 to KS2",
        "type": "score",
        "category": "progress"
    },
    # Average Scores
    "reading_avg_score": {
        "name": "Reading Average Score",
        "short_name": "Reading Avg",
        "description": "Average scaled score in reading",
        "type": "score",
        "category": "average"
    },
    "maths_avg_score": {
        "name": "Maths Average Score",
        "short_name": "Maths Avg",
        "description": "Average scaled score in maths",
        "type": "score",
        "category": "average"
    },
    "gps_avg_score": {
        "name": "GPS Average Score",
        "short_name": "GPS Avg",
        "description": "Average scaled score in GPS",
        "type": "score",
        "category": "average"
    },
    # Gender Performance
    "rwm_expected_boys_pct": {
        "name": "RWM Expected % (Boys)",
        "short_name": "Boys RWM %",
        "description": "% of boys meeting expected standard",
        "type": "percentage",
        "category": "gender"
    },
    "rwm_expected_girls_pct": {
        "name": "RWM Expected % (Girls)",
        "short_name": "Girls RWM %",
        "description": "% of girls meeting expected standard",
        "type": "percentage",
        "category": "gender"
    },
    "rwm_high_boys_pct": {
        "name": "RWM Higher % (Boys)",
        "short_name": "Boys Higher %",
        "description": "% of boys at higher standard",
        "type": "percentage",
        "category": "gender"
    },
    "rwm_high_girls_pct": {
        "name": "RWM Higher % (Girls)",
        "short_name": "Girls Higher %",
        "description": "% of girls at higher standard",
        "type": "percentage",
        "category": "gender"
    },
    # Disadvantaged Performance
    "rwm_expected_disadvantaged_pct": {
        "name": "RWM Expected % (Disadvantaged)",
        "short_name": "Disadvantaged %",
        "description": "% of disadvantaged pupils meeting expected",
        "type": "percentage",
        "category": "equity"
    },
    "rwm_expected_non_disadvantaged_pct": {
        "name": "RWM Expected % (Non-Disadvantaged)",
        "short_name": "Non-Disadv %",
        "description": "% of non-disadvantaged pupils meeting expected",
        "type": "percentage",
        "category": "equity"
    },
    "disadvantaged_gap": {
        "name": "Disadvantaged Gap",
        "short_name": "Disadv Gap",
        "description": "Gap between disadvantaged and national non-disadvantaged",
        "type": "score",
        "category": "equity"
    },
    # School Context
    "disadvantaged_pct": {
        "name": "% Disadvantaged Pupils",
        "short_name": "% Disadvantaged",
        "description": "% of pupils eligible for free school meals or looked after",
        "type": "percentage",
        "category": "context"
    },
    "eal_pct": {
        "name": "% EAL Pupils",
        "short_name": "% EAL",
        "description": "% of pupils with English as additional language",
        "type": "percentage",
        "category": "context"
    },
    "sen_support_pct": {
        "name": "% SEN Support",
        "short_name": "% SEN",
        "description": "% of pupils with SEN support",
        "type": "percentage",
        "category": "context"
    },
    "stability_pct": {
        "name": "% Pupil Stability",
        "short_name": "% Stable",
        "description": "% of non-mobile pupils (stayed at school)",
        "type": "percentage",
        "category": "context"
    },
    # 3-Year Averages
    "rwm_expected_3yr_pct": {
        "name": "RWM Expected % (3-Year Avg)",
        "short_name": "RWM 3yr %",
        "description": "3-year average % meeting expected",
        "type": "percentage",
        "category": "trends"
    },
    "reading_avg_3yr": {
        "name": "Reading Score (3-Year Avg)",
        "short_name": "Reading 3yr",
        "description": "3-year average reading score",
        "type": "score",
        "category": "trends"
    },
    "maths_avg_3yr": {
        "name": "Maths Score (3-Year Avg)",
        "short_name": "Maths 3yr",
        "description": "3-year average maths score",
        "type": "score",
        "category": "trends"
    },
}

# Ranking columns to include in rankings response
RANKING_COLUMNS = [
    "urn", "school_name", "local_authority", "school_type", "address", "year", "total_pupils",
    # Core expected
    "rwm_expected_pct", "reading_expected_pct", "writing_expected_pct", "maths_expected_pct",
    "gps_expected_pct", "science_expected_pct",
    # Core higher
    "rwm_high_pct", "reading_high_pct", "writing_high_pct", "maths_high_pct", "gps_high_pct",
    # Progress & averages
    "reading_progress", "writing_progress", "maths_progress",
    "reading_avg_score", "maths_avg_score", "gps_avg_score",
    # Gender
    "rwm_expected_boys_pct", "rwm_expected_girls_pct", "rwm_high_boys_pct", "rwm_high_girls_pct",
    # Equity
    "rwm_expected_disadvantaged_pct", "rwm_expected_non_disadvantaged_pct", "disadvantaged_gap",
    # Context
    "disadvantaged_pct", "eal_pct", "sen_support_pct", "stability_pct",
    # 3-year
    "rwm_expected_3yr_pct", "reading_avg_3yr", "maths_avg_3yr",
]

# School listing columns
SCHOOL_COLUMNS = ["urn", "school_name", "local_authority", "school_type", "address", "town", "postcode"]

# Local Authority code to name mapping (for fallback when LANAME column missing)
# Source: https://www.gov.uk/government/publications/local-authority-codes
LA_CODE_TO_NAME = {
    # Inner London
    201: "City of London", 202: "Camden", 203: "Greenwich", 204: "Hackney",
    205: "Hammersmith and Fulham", 206: "Islington", 207: "Kensington and Chelsea",
    208: "Lambeth", 209: "Lewisham", 210: "Southwark", 211: "Tower Hamlets",
    212: "Wandsworth", 213: "Westminster",
    # Outer London
    301: "Barking and Dagenham", 302: "Barnet", 303: "Bexley", 304: "Brent",
    305: "Bromley", 306: "Croydon", 307: "Ealing", 308: "Enfield", 309: "Haringey",
    310: "Harrow", 311: "Havering", 312: "Hillingdon", 313: "Hounslow",
    314: "Kingston upon Thames", 315: "Merton", 316: "Newham", 317: "Redbridge",
    318: "Richmond upon Thames", 319: "Sutton", 320: "Waltham Forest",
    # West Midlands
    330: "Birmingham", 331: "Coventry", 332: "Dudley", 333: "Sandwell",
    334: "Solihull", 335: "Walsall", 336: "Wolverhampton",
    # Merseyside
    340: "Knowsley", 341: "Liverpool", 342: "St. Helens", 343: "Sefton", 344: "Wirral",
    # Greater Manchester
    350: "Bolton", 351: "Bury", 352: "Manchester", 353: "Oldham", 354: "Rochdale",
    355: "Salford", 356: "Stockport", 357: "Tameside", 358: "Trafford", 359: "Wigan",
    # South Yorkshire
    370: "Barnsley", 371: "Doncaster", 372: "Rotherham", 373: "Sheffield",
    # West Yorkshire
    380: "Bradford", 381: "Calderdale", 382: "Kirklees", 383: "Leeds", 384: "Wakefield",
    # Tyne and Wear
    390: "Gateshead", 391: "Newcastle upon Tyne", 392: "North Tyneside",
    393: "South Tyneside", 394: "Sunderland",
    # Isles of Scilly
    420: "Isles of Scilly",
    # Unitary authorities (800+)
    800: "Bath and North East Somerset", 801: "Bristol, City of", 802: "North Somerset",
    803: "South Gloucestershire", 805: "Hartlepool", 806: "Middlesbrough",
    807: "Redcar and Cleveland", 808: "Stockton-on-Tees",
    810: "Kingston Upon Hull, City of", 811: "East Riding of Yorkshire",
    812: "North East Lincolnshire", 813: "North Lincolnshire",
    815: "North Yorkshire", 816: "York",
    820: "Bedford", 821: "Central Bedfordshire", 822: "Luton",
    823: "West Northamptonshire", 824: "North Northamptonshire",
    825: "Buckinghamshire", 826: "Milton Keynes",
    830: "Derbyshire", 831: "Derby",
    835: "Dorset", 836: "Bournemouth, Christchurch and Poole",
    837: "Poole", 838: "Bournemouth",  # Historic codes (merged into 836)
    839: "Durham", 840: "Darlington",
    841: "East Sussex", 845: "Brighton and Hove",
    846: "Hampshire", 850: "Portsmouth", 851: "Southampton", 852: "Isle of Wight",
    855: "Leicestershire", 856: "Leicester", 857: "Rutland",
    860: "Staffordshire", 861: "Stoke-on-Trent",
    865: "Wiltshire", 866: "Swindon",
    867: "Bracknell Forest", 868: "Windsor and Maidenhead", 869: "West Berkshire",
    870: "Reading", 871: "Slough", 872: "Wokingham",
    873: "Cambridgeshire", 874: "Peterborough",
    876: "Halton", 877: "Warrington",
    878: "Devon", 879: "Plymouth", 880: "Torbay",
    881: "Essex", 882: "Southend-on-Sea", 883: "Thurrock",
    884: "Herefordshire", 885: "Worcestershire",
    886: "Kent", 887: "Medway",
    888: "Lancashire", 889: "Blackburn with Darwen", 890: "Blackpool",
    891: "Nottinghamshire", 892: "Nottingham",
    893: "Shropshire", 894: "Telford and Wrekin",
    895: "Cheshire East", 896: "Cheshire West and Chester",
    # County councils (900+)
    908: "Cornwall", 909: "Cumbria",
    916: "Gloucestershire", 919: "Hertfordshire",
    921: "Norfolk", 925: "Lincolnshire",
    926: "Northamptonshire",  # Historic (split into 823/824 in 2021)
    928: "Northumberland", 929: "Oxfordshire",
    931: "Somerset", 933: "Suffolk", 935: "Surrey",
    936: "Warwickshire", 937: "West Sussex",
    # New authorities (2023 reorganization)
    938: "Westmorland and Furness", 940: "Cumberland",
    941: "North Yorkshire",  # New unitary
    942: "Somerset",  # New unitary (replaced 931)
    943: "Buckinghamshire",  # New unitary (2020, replacing 825 in some datasets)
}