From c576bba06a3c5b1e15408d274bac9b46a780b46f Mon Sep 17 00:00:00 2001
From: Tudor <tudor@sitaru.org>
Date: Thu, 26 Mar 2026 13:45:23 +0000
Subject: [PATCH] fix(meltano): remove catalog capability and switch elt to run

The `catalog` capability forced Meltano to run --discover and generate
a catalog file (tap.properties.json) before each extraction. This fails
because our Singer SDK taps emit schemas inline and don't need external
catalog files. Removing the capability makes Meltano invoke taps
directly without catalog generation.

Also switch from deprecated `meltano elt` to `meltano run` for
Meltano 4.x compatibility.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 pipeline/dags/school_data_pipeline.py |  6 +++---
 pipeline/meltano.yml                  | 17 -----------------
 2 files changed, 3 insertions(+), 20 deletions(-)

diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py
index 12551c1..0dac821 100644
--- a/pipeline/dags/school_data_pipeline.py
+++ b/pipeline/dags/school_data_pipeline.py
@@ -50,7 +50,7 @@ with DAG(
     with TaskGroup("extract") as extract_group:
         extract_gias = BashOperator(
             task_id="extract_gias",
-            bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} elt tap-uk-gias target-postgres",
+            bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-gias target-postgres",
         )
 
     validate_raw = BashOperator(
@@ -114,7 +114,7 @@ with DAG(
 
     extract_ofsted = BashOperator(
         task_id="extract_ofsted",
-        bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} elt tap-uk-ofsted target-postgres",
+        bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ofsted target-postgres",
     )
 
     dbt_build_ofsted = BashOperator(
@@ -145,7 +145,7 @@ with DAG(
     with TaskGroup("extract_ees") as extract_ees_group:
         extract_ees = BashOperator(
             task_id="extract_ees",
-            bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} elt tap-uk-ees target-postgres",
+            bash_command=f"cd {PIPELINE_DIR} && {MELTANO_BIN} run tap-uk-ees target-postgres",
         )
 
     dbt_build_ees = BashOperator(
diff --git a/pipeline/meltano.yml b/pipeline/meltano.yml
index 087a13e..805e1d2 100644
--- a/pipeline/meltano.yml
+++ b/pipeline/meltano.yml
@@ -7,22 +7,15 @@ plugins:
       namespace: uk_gias
       pip_url: ./plugins/extractors/tap-uk-gias
       executable: tap-uk-gias
-      capabilities:
-        - catalog
-        - state
       settings:
         - name: download_url
           kind: string
           description: GIAS bulk CSV download URL
-          value: https://ea-edubase-api-prod.azurewebsites.net/edubase/downloads/public/edubasealldata.csv
 
     - name: tap-uk-ees
       namespace: uk_ees
       pip_url: ./plugins/extractors/tap-uk-ees
       executable: tap-uk-ees
-      capabilities:
-        - catalog
-        - state
       settings:
         - name: base_url
           kind: string
@@ -35,9 +28,6 @@ plugins:
       namespace: uk_ofsted
       pip_url: ./plugins/extractors/tap-uk-ofsted
       executable: tap-uk-ofsted
-      capabilities:
-        - catalog
-        - state
       settings:
         - name: mi_url
           kind: string
@@ -47,16 +37,11 @@ plugins:
       namespace: uk_parent_view
       pip_url: ./plugins/extractors/tap-uk-parent-view
       executable: tap-uk-parent-view
-      capabilities:
-        - catalog
 
     - name: tap-uk-fbit
       namespace: uk_fbit
       pip_url: ./plugins/extractors/tap-uk-fbit
       executable: tap-uk-fbit
-      capabilities:
-        - catalog
-        - state
       settings:
         - name: base_url
           kind: string
@@ -66,8 +51,6 @@ plugins:
       namespace: uk_idaci
       pip_url: ./plugins/extractors/tap-uk-idaci
       executable: tap-uk-idaci
-      capabilities:
-        - catalog
 
   loaders:
     - name: target-postgres