diff --git a/pipeline/transform/macros/safe_numeric.sql b/pipeline/transform/macros/safe_numeric.sql index 5f14cf1..a1d8d29 100644 --- a/pipeline/transform/macros/safe_numeric.sql +++ b/pipeline/transform/macros/safe_numeric.sql @@ -3,7 +3,9 @@ Casts a string column to numeric, treating any non-numeric value as NULL. Handles all EES suppression codes (z, c, x, q, u, etc.) without needing an explicit list — any string that doesn't look like a number becomes NULL. + A trailing percent sign is accepted: DfE's 2023/24 KS2 information file + writes percentages as "34%" (audit C2). #} {% macro safe_numeric(col) -%} - CASE WHEN {{ col }} ~ '^-?[0-9]+(\.[0-9]+)?$' THEN {{ col }}::numeric ELSE NULL END + CASE WHEN {{ col }} ~ '^-?[0-9]+(\.[0-9]+)?%?$' THEN rtrim({{ col }}, '%')::numeric ELSE NULL END {%- endmacro %} diff --git a/pipeline/transform/models/staging/stg_ees_ks2.yml b/pipeline/transform/models/staging/stg_ees_ks2.yml new file mode 100644 index 0000000..59c9ecd --- /dev/null +++ b/pipeline/transform/models/staging/stg_ees_ks2.yml @@ -0,0 +1,22 @@ +version: 2 + +unit_tests: + - name: stg_ees_ks2_reads_percentages_written_with_a_sign + description: > + DfE's 2023/24 KS2 information file writes percentages as "34%", and every + 2023/24 percentage loaded as null (audit C2). Plain numbers still load; + suppression codes stay null. School 147411's 2023/24 figures are DfE's. + model: stg_ees_ks2 + given: + - input: source('raw', 'ees_ks2_attainment') + rows: + - {school_urn: '147411', time_period: '202324', subject: 'Reading, writing and maths', breakdown_topic: 'All pupils', breakdown: 'Total', expected_standard_pupil_percent: '70'} + - {school_urn: '100000', time_period: '202425', subject: 'Reading, writing and maths', breakdown_topic: 'All pupils', breakdown: 'Total', expected_standard_pupil_percent: '79'} + - input: source('raw', 'ees_ks2_info') + rows: + - {school_urn: '147411', time_period: '202324', totpups: '818', telig: '112', ptfsm6cla1a: '34%', ptealgrp2: '56%', psenelk: '20%', psenele: 'c', ptmobn: '87.5%'} + - {school_urn: '100000', time_period: '202425', totpups: '792', telig: '105', ptfsm6cla1a: '32', ptealgrp2: 'x', psenelk: '14', psenele: '2', ptmobn: '87'} + expect: + rows: + - {urn: 147411, year: 202324, total_pupils: 818, rwm_expected_pct: 70, disadvantaged_pct: 34, eal_pct: 56, sen_support_pct: 20, sen_ehcp_pct: null, stability_pct: 87.5} + - {urn: 100000, year: 202425, total_pupils: 792, rwm_expected_pct: 79, disadvantaged_pct: 32, eal_pct: null, sen_support_pct: 14, sen_ehcp_pct: 2, stability_pct: 87}