diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 12551c1..0dac821 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -50,7 +50,7 @@ with DAG( with TaskGroup("extract") as extract_group: extract_gias = BashOperator( task_id="extract_gias", - bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} elt tap-uk-gias target-postgres", + bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-gias target-postgres", ) validate_raw = BashOperator( @@ -114,7 +114,7 @@ with DAG( extract_ofsted = BashOperator( task_id="extract_ofsted", - bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} elt tap-uk-ofsted target-postgres", + bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ofsted target-postgres", ) dbt_build_ofsted = BashOperator( @@ -145,7 +145,7 @@ with DAG( with TaskGroup("extract_ees") as extract_ees_group: extract_ees = BashOperator( task_id="extract_ees", - bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} elt tap-uk-ees target-postgres", + bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ees target-postgres", ) dbt_build_ees = BashOperator( diff --git a/pipeline/meltano.yml b/pipeline/meltano.yml index 087a13e..805e1d2 100644 --- a/pipeline/meltano.yml +++ b/pipeline/meltano.yml @@ -7,22 +7,15 @@ plugins: namespace: uk_gias pip_url: ./plugins/extractors/tap-uk-gias executable: tap-uk-gias - capabilities: - - catalog - - state settings: - name: download_url kind: string description: GIAS bulk CSV download URL - value: https://ea-edubase-api-prod.azurewebsites.net/edubase/downloads/public/edubasealldata.csv - name: tap-uk-ees namespace: uk_ees pip_url: ./plugins/extractors/tap-uk-ees executable: tap-uk-ees - capabilities: - - catalog - - state settings: - name: base_url kind: string @@ -35,9 +28,6 @@ plugins: namespace: uk_ofsted pip_url: ./plugins/extractors/tap-uk-ofsted executable: tap-uk-ofsted - capabilities: - - catalog - - state settings: - name: mi_url kind: string @@ -47,16 +37,11 @@ plugins: namespace: uk_parent_view pip_url: ./plugins/extractors/tap-uk-parent-view executable: tap-uk-parent-view - capabilities: - - catalog - name: tap-uk-fbit namespace: uk_fbit pip_url: ./plugins/extractors/tap-uk-fbit executable: tap-uk-fbit - capabilities: - - catalog - - state settings: - name: base_url kind: string @@ -66,8 +51,6 @@ plugins: namespace: uk_idaci pip_url: ./plugins/extractors/tap-uk-idaci executable: tap-uk-idaci - capabilities: - - catalog loaders: - name: target-postgres