feat: migrate backend to marts schema, update EES tap for verified datasets
Pipeline: - EES tap: split KS4 into performance + info streams, fix admissions filename (SchoolLevel keyword match), fix census filename (yearly suffix), remove phonics (no school-level data on EES), change endswith → in for matching - stg_ees_ks4: rewrite to filter long-format data and extract Attainment 8, Progress 8, EBacc, English/Maths metrics; join KS4 info for context - stg_ees_admissions: map real CSV columns (total_number_places_offered, etc.) - stg_ees_census: update source reference, stub with TODO for data columns - Remove stg_ees_phonics, fact_phonics (no school-level EES data) - Add ees_ks4_performance + ees_ks4_info sources, remove ees_ks4 + ees_phonics - Update int_ks4_with_lineage + fact_ks4_performance with new KS4 columns - Annual EES DAG: remove stg_ees_phonics+ from selector Backend: - models.py: replace all models to point at marts.* tables with schema='marts' (DimSchool, DimLocation, KS2Performance, FactOfstedInspection, etc.) - data_loader.py: rewrite load_school_data_as_dataframe() using raw SQL joining dim_school + dim_location + fact_ks2_performance; update get_supplementary_data() - database.py: remove migration machinery, keep only connection setup - app.py: remove check_and_migrate_if_needed, remove /api/admin/reimport-ks2 endpoints (pipeline handles all imports) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,16 +4,14 @@ with current_ks4 as (
|
||||
select
|
||||
urn as current_urn,
|
||||
urn as source_urn,
|
||||
year,
|
||||
total_pupils,
|
||||
progress_8_score,
|
||||
year, total_pupils, eligible_pupils, prior_attainment_avg,
|
||||
attainment_8_score,
|
||||
ebacc_entry_pct,
|
||||
ebacc_achievement_pct,
|
||||
english_strong_pass_pct,
|
||||
maths_strong_pass_pct,
|
||||
english_maths_strong_pass_pct,
|
||||
staying_in_education_pct
|
||||
progress_8_score, progress_8_lower_ci, progress_8_upper_ci,
|
||||
progress_8_english, progress_8_maths, progress_8_ebacc, progress_8_open,
|
||||
english_maths_strong_pass_pct, english_maths_standard_pass_pct,
|
||||
ebacc_entry_pct, ebacc_strong_pass_pct, ebacc_standard_pass_pct, ebacc_avg_score,
|
||||
gcse_grade_91_pct,
|
||||
sen_pct, sen_ehcp_pct, sen_support_pct
|
||||
from {{ ref('stg_ees_ks4') }}
|
||||
),
|
||||
|
||||
@@ -21,16 +19,14 @@ predecessor_ks4 as (
|
||||
select
|
||||
lin.current_urn,
|
||||
ks4.urn as source_urn,
|
||||
ks4.year,
|
||||
ks4.total_pupils,
|
||||
ks4.progress_8_score,
|
||||
ks4.year, ks4.total_pupils, ks4.eligible_pupils, ks4.prior_attainment_avg,
|
||||
ks4.attainment_8_score,
|
||||
ks4.ebacc_entry_pct,
|
||||
ks4.ebacc_achievement_pct,
|
||||
ks4.english_strong_pass_pct,
|
||||
ks4.maths_strong_pass_pct,
|
||||
ks4.english_maths_strong_pass_pct,
|
||||
ks4.staying_in_education_pct
|
||||
ks4.progress_8_score, ks4.progress_8_lower_ci, ks4.progress_8_upper_ci,
|
||||
ks4.progress_8_english, ks4.progress_8_maths, ks4.progress_8_ebacc, ks4.progress_8_open,
|
||||
ks4.english_maths_strong_pass_pct, ks4.english_maths_standard_pass_pct,
|
||||
ks4.ebacc_entry_pct, ks4.ebacc_strong_pass_pct, ks4.ebacc_standard_pass_pct, ks4.ebacc_avg_score,
|
||||
ks4.gcse_grade_91_pct,
|
||||
ks4.sen_pct, ks4.sen_ehcp_pct, ks4.sen_support_pct
|
||||
from {{ ref('stg_ees_ks4') }} ks4
|
||||
inner join {{ ref('int_school_lineage') }} lin
|
||||
on ks4.urn = lin.predecessor_urn
|
||||
|
||||
@@ -1,18 +1,8 @@
|
||||
-- Intermediate model: Merged pupil characteristics from census data
|
||||
-- TODO: Expand once census data columns are verified and added to stg_ees_census
|
||||
|
||||
select
|
||||
urn,
|
||||
year,
|
||||
fsm_pct,
|
||||
sen_support_pct,
|
||||
sen_ehcp_pct,
|
||||
eal_pct,
|
||||
disadvantaged_pct,
|
||||
ethnicity_white_pct,
|
||||
ethnicity_asian_pct,
|
||||
ethnicity_black_pct,
|
||||
ethnicity_mixed_pct,
|
||||
ethnicity_other_pct,
|
||||
class_size_avg,
|
||||
stability_pct
|
||||
phase_type_grouping
|
||||
from {{ ref('stg_ees_census') }}
|
||||
|
||||
Reference in New Issue
Block a user