From deb402473183afd8f3d88007096e6c71e41391c0 Mon Sep 17 00:00:00 2001 From: Tudor Date: Thu, 26 Mar 2026 09:18:11 +0000 Subject: [PATCH] chore(pipeline): bump all dependencies to latest stable versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Airflow 2.11 → 3.1 (BashOperator moved to providers-standard) - Meltano 3.5 → 4.1 (meltano.yml version 2, meltanolabs target-postgres) - dbt-postgres 1.9 → 1.10 - singer-sdk 0.39 → 0.53 (all 6 taps) - Typesense Docker 27.1 → 30.1 - Typesense Python client >=2.0 - Python base image 3.12 → 3.13 Co-Authored-By: Claude Opus 4.6 --- docker-compose.portainer.yml | 2 +- docker-compose.yml | 2 +- pipeline/Dockerfile | 2 +- pipeline/dags/school_data_pipeline.py | 3 +-- pipeline/meltano.yml | 8 ++++---- pipeline/plugins/extractors/tap-uk-ees/pyproject.toml | 2 +- pipeline/plugins/extractors/tap-uk-fbit/pyproject.toml | 2 +- pipeline/plugins/extractors/tap-uk-gias/pyproject.toml | 2 +- pipeline/plugins/extractors/tap-uk-idaci/pyproject.toml | 2 +- pipeline/plugins/extractors/tap-uk-ofsted/pyproject.toml | 2 +- .../plugins/extractors/tap-uk-parent-view/pyproject.toml | 2 +- pipeline/requirements.txt | 9 +++++---- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml index 5bfc35e..b8d22ce 100644 --- a/docker-compose.portainer.yml +++ b/docker-compose.portainer.yml @@ -86,7 +86,7 @@ services: # ── Typesense Search Engine ─────────────────────────────────────────── typesense: - image: typesense/typesense:27.1 + image: typesense/typesense:30.1 container_name: schoolcompare_typesense environment: TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:-changeme} diff --git a/docker-compose.yml b/docker-compose.yml index cba4b9c..42fbf5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,7 @@ services: # Typesense — search engine typesense: - image: typesense/typesense:27.1 + image: typesense/typesense:30.1 container_name: schoolcompare_typesense ports: - "8108:8108" diff --git a/pipeline/Dockerfile b/pipeline/Dockerfile index 9562efd..4b8a4da 100644 --- a/pipeline/Dockerfile +++ b/pipeline/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.13-slim WORKDIR /opt/pipeline diff --git a/pipeline/dags/school_data_pipeline.py b/pipeline/dags/school_data_pipeline.py index 1771ac9..6829819 100644 --- a/pipeline/dags/school_data_pipeline.py +++ b/pipeline/dags/school_data_pipeline.py @@ -15,8 +15,7 @@ from __future__ import annotations from datetime import datetime, timedelta from airflow import DAG -from airflow.operators.bash import BashOperator -from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup PIPELINE_DIR = "/opt/pipeline" diff --git a/pipeline/meltano.yml b/pipeline/meltano.yml index 58aa838..087a13e 100644 --- a/pipeline/meltano.yml +++ b/pipeline/meltano.yml @@ -1,4 +1,4 @@ -version: 1 +version: 2 project_id: school-compare-pipeline plugins: @@ -71,8 +71,8 @@ plugins: loaders: - name: target-postgres - variant: transferwise - pip_url: pipelinewise-target-postgres + variant: meltanolabs + pip_url: meltanolabs-target-postgres config: host: $PG_HOST port: $PG_PORT @@ -84,7 +84,7 @@ plugins: utilities: - name: dbt-postgres variant: dbt-labs - pip_url: dbt-postgres~=1.9 + pip_url: dbt-postgres~=1.10 config: project_dir: $MELTANO_PROJECT_ROOT/transform profiles_dir: $MELTANO_PROJECT_ROOT/transform diff --git a/pipeline/plugins/extractors/tap-uk-ees/pyproject.toml b/pipeline/plugins/extractors/tap-uk-ees/pyproject.toml index 681766e..bef967e 100644 --- a/pipeline/plugins/extractors/tap-uk-ees/pyproject.toml +++ b/pipeline/plugins/extractors/tap-uk-ees/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Singer tap for UK Explore Education Statistics (KS2, KS4, Census, Admissions, Phonics)" requires-python = ">=3.10" dependencies = [ - "singer-sdk~=0.39", + "singer-sdk~=0.53", "requests>=2.31", "pandas>=2.0", ] diff --git a/pipeline/plugins/extractors/tap-uk-fbit/pyproject.toml b/pipeline/plugins/extractors/tap-uk-fbit/pyproject.toml index 890479f..eda9a3a 100644 --- a/pipeline/plugins/extractors/tap-uk-fbit/pyproject.toml +++ b/pipeline/plugins/extractors/tap-uk-fbit/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Singer tap for UK FBIT (Financial Benchmarking and Insights Tool)" requires-python = ">=3.10" dependencies = [ - "singer-sdk~=0.39", + "singer-sdk~=0.53", "requests>=2.31", ] diff --git a/pipeline/plugins/extractors/tap-uk-gias/pyproject.toml b/pipeline/plugins/extractors/tap-uk-gias/pyproject.toml index c748c85..2643c2b 100644 --- a/pipeline/plugins/extractors/tap-uk-gias/pyproject.toml +++ b/pipeline/plugins/extractors/tap-uk-gias/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Singer tap for UK GIAS (Get Information About Schools) bulk data" requires-python = ">=3.10" dependencies = [ - "singer-sdk~=0.39", + "singer-sdk~=0.53", "requests>=2.31", "pandas>=2.0", ] diff --git a/pipeline/plugins/extractors/tap-uk-idaci/pyproject.toml b/pipeline/plugins/extractors/tap-uk-idaci/pyproject.toml index 26f69a3..ce319f7 100644 --- a/pipeline/plugins/extractors/tap-uk-idaci/pyproject.toml +++ b/pipeline/plugins/extractors/tap-uk-idaci/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Singer tap for UK IDACI deprivation index" requires-python = ">=3.10" dependencies = [ - "singer-sdk~=0.39", + "singer-sdk~=0.53", "requests>=2.31", "pandas>=2.0", ] diff --git a/pipeline/plugins/extractors/tap-uk-ofsted/pyproject.toml b/pipeline/plugins/extractors/tap-uk-ofsted/pyproject.toml index 3474a0f..6352a4f 100644 --- a/pipeline/plugins/extractors/tap-uk-ofsted/pyproject.toml +++ b/pipeline/plugins/extractors/tap-uk-ofsted/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Singer tap for UK Ofsted Management Information" requires-python = ">=3.10" dependencies = [ - "singer-sdk~=0.39", + "singer-sdk~=0.53", "requests>=2.31", "pandas>=2.0", "odfpy>=1.4", diff --git a/pipeline/plugins/extractors/tap-uk-parent-view/pyproject.toml b/pipeline/plugins/extractors/tap-uk-parent-view/pyproject.toml index 4d45bfa..d804953 100644 --- a/pipeline/plugins/extractors/tap-uk-parent-view/pyproject.toml +++ b/pipeline/plugins/extractors/tap-uk-parent-view/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Singer tap for UK Ofsted Parent View survey data" requires-python = ">=3.10" dependencies = [ - "singer-sdk~=0.39", + "singer-sdk~=0.53", "requests>=2.31", "pandas>=2.0", "openpyxl>=3.1", diff --git a/pipeline/requirements.txt b/pipeline/requirements.txt index 246b9c3..3ae51ce 100644 --- a/pipeline/requirements.txt +++ b/pipeline/requirements.txt @@ -1,9 +1,10 @@ # Pipeline dependencies -meltano==3.5.* -dbt-postgres~=1.9 -apache-airflow==2.11.* +meltano==4.1.* +dbt-postgres~=1.10 +apache-airflow==3.1.* +apache-airflow-providers-standard>=1.0 apache-airflow-providers-postgres>=5.0 -typesense>=0.21 +typesense>=2.0,<3 requests>=2.31 openpyxl>=3.1 odfpy>=1.4