diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 52a37bd..c4db5e3 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -149,3 +149,53 @@ with DAG( ) extract_ees_group >> dbt_build_ees >> sync_typesense_ees + + +# ── Monthly DAG (Parent View) ────────────────────────────────────────── + +with DAG( + dag_id="school_data_monthly_parent_view", + default_args=default_args, + description="Monthly Ofsted Parent View extraction and transform", + schedule="0 3 1 * *", + start_date=datetime(2025, 1, 1), + catchup=False, + tags=["school-compare", "monthly"], +) as monthly_parent_view_dag: + + extract_parent_view = BashOperator( + task_id="extract_parent_view", + bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-parent-view target-postgres", + ) + + dbt_build_parent_view = BashOperator( + task_id="dbt_build", + bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_parent_view+ fact_parent_view+", + ) + + extract_parent_view >> dbt_build_parent_view + + +# ── Annual DAG (IDACI Deprivation) ──────────────────────────────────── + +with DAG( + dag_id="school_data_annual_idaci", + default_args=default_args, + description="Annual IDACI deprivation index extraction and transform", + schedule=None, # Triggered manually when new IDACI release is published + start_date=datetime(2025, 1, 1), + catchup=False, + tags=["school-compare", "annual"], +) as annual_idaci_dag: + + extract_idaci = BashOperator( + task_id="extract_idaci", + bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-idaci target-postgres", + ) + + dbt_build_idaci = BashOperator( + task_id="dbt_build", + bash_command=f"cd {PIPELINE_DIR}/transform && {DBT_BIN} build --profiles-dir . --target production --select stg_idaci+ fact_deprivation+", + ) + + extract_idaci >> dbt_build_idaci