diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6d72959..23ae57c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -61,6 +61,11 @@ history from the full DataFrame and supplementary data from marts. Comparisons batch supplementary queries across selected URNs. Async routes still contain synchronous dependency calls; a fully asynchronous database layer is not present. +Benchmarks are DfE's own figures, read from marts: `fact_ks2_national_averages` +and `fact_ks4_national_averages` for England, and `fact_ks4_la_averages` (all +state-funded schools) for the search rows' "vs LA avg". The API never averages +school rows to make a benchmark. + ## Frontend boundaries `app/(frontend)` owns the public root layout and pages. `app/(payload)` owns the diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 2b27af7..68a683e 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -193,7 +193,7 @@ with DAG( dbt_build_ees = BashOperator( task_id="dbt_build", - bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ees_ks2+ stg_legacy_ks2+ stg_ees_ks4+ stg_legacy_ks4+ stg_ees_census+ stg_ees_admissions+ stg_ees_ks2_national+ stg_ees_ks4_national+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+ stg_ees_ks4_destinations_national+ stg_ees_ks5_destinations_national+", + bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_ees_ks2+ stg_legacy_ks2+ stg_ees_ks4+ stg_legacy_ks4+ stg_ees_census+ stg_ees_admissions+ stg_ees_ks2_national+ stg_ees_ks4_national+ stg_ees_ks4_la+ stg_ees_ks4_destinations+ stg_ees_ks5_destinations+ stg_ees_ks4_destinations_national+ stg_ees_ks5_destinations_national+", ) sync_typesense_ees = BashOperator( diff --git a/pipeline/transform/models/marts/_marts_schema.yml b/pipeline/transform/models/marts/_marts_schema.yml index b198b6b..4355775 100644 --- a/pipeline/transform/models/marts/_marts_schema.yml +++ b/pipeline/transform/models/marts/_marts_schema.yml @@ -279,11 +279,24 @@ models: tests: [not_null, unique] - name: fact_ks4_national_averages - description: Computed national KS4 averages (means across state schools in our dataset — not official DfE figures) — one row per academic year + description: Official DfE KS4 national headline averages (England, all state-funded schools, all pupils) — one row per academic year columns: - name: year tests: [not_null, unique] + - name: fact_ks4_la_averages + description: Official DfE KS4 local-authority averages (all state-funded schools, all pupils) — one row per academic year and LA; la_code is the GIAS LA code + columns: + - name: year + tests: [not_null] + - name: la_code + tests: [not_null] + - name: la_name + tests: [not_null] + tests: + - unique: + column_name: "year || '-' || la_code" + - name: fact_deprivation description: IDACI deprivation index — one row per URN columns: diff --git a/pipeline/transform/models/marts/fact_ks4_la_averages.sql b/pipeline/transform/models/marts/fact_ks4_la_averages.sql new file mode 100644 index 0000000..40ab45c --- /dev/null +++ b/pipeline/transform/models/marts/fact_ks4_la_averages.sql @@ -0,0 +1,22 @@ +{{ config(materialized='table') }} + +-- Mart: OFFICIAL DfE KS4 local-authority averages — one row per academic year +-- and LA (all state-funded schools, all pupils), from the same EES data set +-- as fact_ks4_national_averages. Feeds "vs LA avg" on search rows and map +-- cards, replacing a mean the API took over every school in the LA, +-- independent and special included (audit H2). la_code is the GIAS LA code. + +select + year, + la_code, + la_name, + attainment_8_score, + progress_8_score, + english_maths_standard_pass_pct, + english_maths_strong_pass_pct, + ebacc_entry_pct, + ebacc_standard_pass_pct, + ebacc_strong_pass_pct, + ebacc_avg_score +from {{ ref('stg_ees_ks4_la') }} +order by year, la_code diff --git a/pipeline/transform/models/staging/_stg_sources.yml b/pipeline/transform/models/staging/_stg_sources.yml index 2d5880a..7aa577f 100644 --- a/pipeline/transform/models/staging/_stg_sources.yml +++ b/pipeline/transform/models/staging/_stg_sources.yml @@ -79,6 +79,9 @@ sources: - name: ees_ks4_national description: Official KS4 national headline averages from DfE EES data catalogue — one row per academic year + - name: ees_ks4_la + description: Official KS4 local-authority averages (all state-funded schools) from the same DfE EES data set as ees_ks4_national — one row per academic year and LA + # Phonics: no school-level data on EES (only national/LA level) - name: fbit_finance diff --git a/pipeline/transform/models/staging/stg_ees_ks4_la.sql b/pipeline/transform/models/staging/stg_ees_ks4_la.sql new file mode 100644 index 0000000..225ce7e --- /dev/null +++ b/pipeline/transform/models/staging/stg_ees_ks4_la.sql @@ -0,0 +1,21 @@ +-- Staging model: official DfE KS4 local-authority averages — one row per +-- academic year and LA (all state-funded schools, all pupils). Same EES data +-- set as stg_ees_ks4_national. la_code is the GIAS LA code +-- (dim_location.local_authority_code). Suppressed values ('z', 'x') are +-- coerced to NULL by safe_numeric. + +select + cast(trim(time_period) as integer) as year, + cast(trim(old_la_code) as integer) as la_code, + trim(la_name) as la_name, + {{ safe_numeric('attainment_8_score') }} as attainment_8_score, + {{ safe_numeric('progress_8_score') }} as progress_8_score, + {{ safe_numeric('english_maths_standard_pass_pct') }} as english_maths_standard_pass_pct, + {{ safe_numeric('english_maths_strong_pass_pct') }} as english_maths_strong_pass_pct, + {{ safe_numeric('ebacc_entry_pct') }} as ebacc_entry_pct, + {{ safe_numeric('ebacc_standard_pass_pct') }} as ebacc_standard_pass_pct, + {{ safe_numeric('ebacc_strong_pass_pct') }} as ebacc_strong_pass_pct, + {{ safe_numeric('ebacc_avg_score') }} as ebacc_avg_score +from {{ source('raw', 'ees_ks4_la') }} +where time_period ~ '^[0-9]+$' + and old_la_code ~ '^[0-9]+$' diff --git a/pipeline/transform/models/staging/stg_ees_ks4_la.yml b/pipeline/transform/models/staging/stg_ees_ks4_la.yml new file mode 100644 index 0000000..6e5b52b --- /dev/null +++ b/pipeline/transform/models/staging/stg_ees_ks4_la.yml @@ -0,0 +1,15 @@ +version: 2 + +unit_tests: + - name: stg_ees_ks4_la_casts_codes_years_and_measures + description: DfE's LA rows, keyed by the GIAS LA code. Suppressed values ('z') become null. + model: stg_ees_ks4_la + given: + - input: source('raw', 'ees_ks4_la') + rows: + - {time_period: '202425', old_la_code: '207', new_la_code: 'E09000020', la_name: 'Kensington and Chelsea', attainment_8_score: '54.5', progress_8_score: 'z', english_maths_standard_pass_pct: '77'} + - {time_period: '202324', old_la_code: '207', new_la_code: 'E09000020', la_name: 'Kensington and Chelsea', attainment_8_score: '54.5', progress_8_score: '0.29', english_maths_standard_pass_pct: '76'} + expect: + rows: + - {year: 202425, la_code: 207, la_name: 'Kensington and Chelsea', attainment_8_score: 54.5, progress_8_score: null, english_maths_standard_pass_pct: 77} + - {year: 202324, la_code: 207, la_name: 'Kensington and Chelsea', attainment_8_score: 54.5, progress_8_score: 0.29, english_maths_standard_pass_pct: 76}