diff --git a/pipeline/transform/dbt_project.yml b/pipeline/transform/dbt_project.yml index 86dce6a..ba23641 100644 --- a/pipeline/transform/dbt_project.yml +++ b/pipeline/transform/dbt_project.yml @@ -23,6 +23,11 @@ models: +materialized: table +schema: marts +tests: + school_compare: + assert_no_orphaned_facts: + +severity: warn + seeds: school_compare: +schema: seeds diff --git a/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql b/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql index e9c4066..68dc193 100644 --- a/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql +++ b/pipeline/transform/models/intermediate/int_ks2_with_lineage.sql @@ -19,7 +19,8 @@ with current_ks2 as ( ), predecessor_ks2 as ( - select + -- If multiple predecessors have data for the same year, keep the one with most pupils. + select distinct on (lin.current_urn, ks2.year) lin.current_urn, ks2.urn as source_urn, ks2.year, ks2.total_pupils, ks2.eligible_pupils, @@ -40,6 +41,7 @@ predecessor_ks2 as ( where curr.urn = lin.current_urn and curr.year = ks2.year ) + order by lin.current_urn, ks2.year, ks2.total_pupils desc nulls last ), combined as (