From 06bf53ac2675d791a0aa6b24a0832350182debfc Mon Sep 17 00:00:00 2001 From: Tudor Sitaru Date: Thu, 9 Apr 2026 14:48:31 +0100 Subject: [PATCH] fix(dag): remove invalid --select flag from meltano run meltano run does not support --select; the full tap-uk-ees run already includes EESKs2NationalStream so no separate task is needed. Co-Authored-By: Claude Sonnet 4.6 --- pipeline/dags/school_data_pipeline.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index a6c5c08..5586512 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -133,15 +133,11 @@ with DAG( ) as annual_ees_dag: with TaskGroup("extract_ees") as extract_ees_group: + # Runs all tap-uk-ees streams, including the new ees_ks2_national stream extract_ees = BashOperator( task_id="extract_ees", bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ees target-postgres", ) - # KS2 national headlines run in parallel — small single-CSV download - extract_ks2_national = BashOperator( - task_id="extract_ks2_national", - bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ees target-postgres --select ees_ks2_national", - ) dbt_build_ees = BashOperator( task_id="dbt_build",