fix(dags): build stg_gias_ukprn monthly, never daily (review)
Rebuilding the view in the daily build dropped the Ofsted views on it (CASCADE), which dim_school reads: every daily run would have failed. A selector test now rejects any build that drops views it then reads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cfc9367bb3
commit
c16dd0044f
3 files changed
+43
-2
No files matched your search
@@ -96,3 +96,41 @@ def test_selected_models_only_read_models_that_exist(dag_id):
|
||||
missing = {model: sorted(PARENTS[model] - available) for model in sorted(selected)
|
||||
if PARENTS[model] - available}
|
||||
assert missing == {}, f'{dag_id} builds models whose parents it never builds: {missing}'
|
||||
|
||||
|
||||
TABLE = re.compile(r"materialized\s*=\s*['\"]table['\"]")
|
||||
|
||||
|
||||
def views():
|
||||
"""Staging and intermediate models are views unless configured as tables."""
|
||||
out = set()
|
||||
for path in MODELS.rglob('*.sql'):
|
||||
layer = path.relative_to(MODELS).parts[0]
|
||||
if layer in ('staging', 'intermediate') and not TABLE.search(path.read_text()):
|
||||
out.add(path.stem)
|
||||
return out
|
||||
|
||||
|
||||
VIEWS = views()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('dag_id', sorted(BUILDS))
|
||||
def test_rebuilt_relations_do_not_drop_views_the_build_reads(dag_id):
|
||||
"""dbt rebuilds a relation by dropping the old one with CASCADE, which drops
|
||||
every view built on it. A view dropped that way and not rebuilt by the same
|
||||
build is gone when that build reads it: the daily GIAS build rebuilt
|
||||
stg_gias_ukprn and dropped the Ofsted views dim_school reads (audit H4)."""
|
||||
selected = selected_models(BUILDS[dag_id], PARENTS)
|
||||
children = defaultdict(set)
|
||||
for model, ps in PARENTS.items():
|
||||
for p in ps:
|
||||
children[p].add(model)
|
||||
dropped, stack = set(), list(selected)
|
||||
while stack:
|
||||
for child in children[stack.pop()]:
|
||||
if child in VIEWS and child not in dropped:
|
||||
dropped.add(child)
|
||||
stack.append(child)
|
||||
dropped -= selected
|
||||
broken = {m: sorted(PARENTS[m] & dropped) for m in sorted(selected) if PARENTS[m] & dropped}
|
||||
assert broken == {}, f'{dag_id} drops views it then reads: {broken}'
|
||||
Reference in new issue
Block a user