fix(dbt): read percentages DfE writes as "34%" (C2)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
242c603aec
commit
4ae3853277
2 files changed
+25
-1
No files matched your search
@@ -3,7 +3,9 @@
|
||||
Casts a string column to numeric, treating any non-numeric value as NULL.
|
||||
Handles all EES suppression codes (z, c, x, q, u, etc.) without needing
|
||||
an explicit list — any string that doesn't look like a number becomes NULL.
|
||||
A trailing percent sign is accepted: DfE's 2023/24 KS2 information file
|
||||
writes percentages as "34%" (audit C2).
|
||||
#}
|
||||
{% macro safe_numeric(col) -%}
|
||||
CASE WHEN {{ col }} ~ '^-?[0-9]+(\.[0-9]+)?$' THEN {{ col }}::numeric ELSE NULL END
|
||||
CASE WHEN {{ col }} ~ '^-?[0-9]+(\.[0-9]+)?%?$' THEN rtrim({{ col }}, '%')::numeric ELSE NULL END
|
||||
{%- endmacro %}
|
||||
@@ -0,0 +1,22 @@
|
||||
version: 2
|
||||
|
||||
unit_tests:
|
||||
- name: stg_ees_ks2_reads_percentages_written_with_a_sign
|
||||
description: >
|
||||
DfE's 2023/24 KS2 information file writes percentages as "34%", and every
|
||||
2023/24 percentage loaded as null (audit C2). Plain numbers still load;
|
||||
suppression codes stay null. School 147411's 2023/24 figures are DfE's.
|
||||
model: stg_ees_ks2
|
||||
given:
|
||||
- input: source('raw', 'ees_ks2_attainment')
|
||||
rows:
|
||||
- {school_urn: '147411', time_period: '202324', subject: 'Reading, writing and maths', breakdown_topic: 'All pupils', breakdown: 'Total', expected_standard_pupil_percent: '70'}
|
||||
- {school_urn: '100000', time_period: '202425', subject: 'Reading, writing and maths', breakdown_topic: 'All pupils', breakdown: 'Total', expected_standard_pupil_percent: '79'}
|
||||
- input: source('raw', 'ees_ks2_info')
|
||||
rows:
|
||||
- {school_urn: '147411', time_period: '202324', totpups: '818', telig: '112', ptfsm6cla1a: '34%', ptealgrp2: '56%', psenelk: '20%', psenele: 'c', ptmobn: '87.5%'}
|
||||
- {school_urn: '100000', time_period: '202425', totpups: '792', telig: '105', ptfsm6cla1a: '32', ptealgrp2: 'x', psenelk: '14', psenele: '2', ptmobn: '87'}
|
||||
expect:
|
||||
rows:
|
||||
- {urn: 147411, year: 202324, total_pupils: 818, rwm_expected_pct: 70, disadvantaged_pct: 34, eal_pct: 56, sen_support_pct: 20, sen_ehcp_pct: null, stability_pct: 87.5}
|
||||
- {urn: 100000, year: 202425, total_pupils: 792, rwm_expected_pct: 79, disadvantaged_pct: 32, eal_pct: null, sen_support_pct: 14, sen_ehcp_pct: 2, stability_pct: 87}
|
||||
Reference in new issue
Block a user