From 3785f0c977faacea8fed79b03193a2c96646673d Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 18 Jun 2026 13:41:35 +0100 Subject: [PATCH] fix(pipeline): invoke dbt via python module to avoid Fusion shadowing The standalone dbt Fusion binary (dbt-core 2.x) on PATH shadows the pip-installed classic dbt-postgres ~=1.10 and rejects the Postgres adapter (dbt1005), breaking every DAG's dbt_build task. Invoke dbt via `python -m dbt.cli.main` in the DAGs and the Dockerfile dbt deps step so the classic Postgres-capable engine is always used regardless of PATH. Co-Authored-By: Claude Opus 4.8 --- pipeline/Dockerfile | 5 +++-- pipeline/dags/school_data_pipeline.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pipeline/Dockerfile b/pipeline/Dockerfile index c498be2..7dee7d2 100644 --- a/pipeline/Dockerfile +++ b/pipeline/Dockerfile @@ -32,8 +32,9 @@ COPY dags/ dags/ # Initialize Meltano project (generates catalog, installs target-postgres) RUN meltano install -# dbt deps -RUN cd transform && dbt deps --profiles-dir . 2>/dev/null || true +# dbt deps — invoke via the Python module so the classic dbt-postgres engine +# is used, not a standalone dbt Fusion binary that may shadow it on PATH. +RUN cd transform && python -m dbt.cli.main deps --profiles-dir . 2>/dev/null || true ENV AIRFLOW_HOME=/opt/airflow ENV AIRFLOW__CORE__DAGS_FOLDER=/opt/pipeline/dags diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index c6423da..e7d79bf 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -24,7 +24,11 @@ except ImportError: PIPELINE_DIR = "/opt/pipeline" MELTANO_BIN = "meltano" -DBT_BIN = "dbt" +# Invoke dbt via the Python module rather than a bare `dbt` on PATH. The +# standalone dbt Fusion binary (dbt-core 2.x) shadows the pip-installed +# classic engine on some images and rejects the Postgres adapter +# (dbt1005). The module form always resolves to dbt-postgres ~=1.10. +DBT_BIN = "python -m dbt.cli.main" default_args = { "owner": "school-compare",