diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index a6c5c08..5586512 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -133,15 +133,11 @@ with DAG( ) as annual_ees_dag: with TaskGroup("extract_ees") as extract_ees_group: + # Runs all tap-uk-ees streams, including the new ees_ks2_national stream extract_ees = BashOperator( task_id="extract_ees", bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ees target-postgres", ) - # KS2 national headlines run in parallel — small single-CSV download - extract_ks2_national = BashOperator( - task_id="extract_ks2_national", - bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ees target-postgres --select ees_ks2_national", - ) dbt_build_ees = BashOperator( task_id="dbt_build",