Tudor
deb4024731
All checks were successful
Build and Push Docker Images / Build Backend (FastAPI) (push) Successful in 32s
Build and Push Docker Images / Build Frontend (Next.js) (push) Successful in 1m4s
Build and Push Docker Images / Build Integrator (push) Successful in 57s
Build and Push Docker Images / Build Kestra Init (push) Successful in 32s
Build and Push Docker Images / Build Pipeline (Meltano + dbt + Airflow) (push) Successful in 1m45s
Build and Push Docker Images / Trigger Portainer Update (push) Successful in 0s
- Airflow 2.11 → 3.1 (BashOperator moved to providers-standard) - Meltano 3.5 → 4.1 (meltano.yml version 2, meltanolabs target-postgres) - dbt-postgres 1.9 → 1.10 - singer-sdk 0.39 → 0.53 (all 6 taps) - Typesense Docker 27.1 → 30.1 - Typesense Python client >=2.0 - Python base image 3.12 → 3.13 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
115 lines
2.9 KiB
YAML
115 lines
2.9 KiB
YAML
version: 2
|
|
project_id: school-compare-pipeline
|
|
|
|
plugins:
|
|
extractors:
|
|
- name: tap-uk-gias
|
|
namespace: uk_gias
|
|
pip_url: ./plugins/extractors/tap-uk-gias
|
|
executable: tap-uk-gias
|
|
capabilities:
|
|
- catalog
|
|
- state
|
|
settings:
|
|
- name: download_url
|
|
kind: string
|
|
description: GIAS bulk CSV download URL
|
|
value: https://ea-edubase-api-prod.azurewebsites.net/edubase/downloads/public/edubasealldata.csv
|
|
|
|
- name: tap-uk-ees
|
|
namespace: uk_ees
|
|
pip_url: ./plugins/extractors/tap-uk-ees
|
|
executable: tap-uk-ees
|
|
capabilities:
|
|
- catalog
|
|
- state
|
|
settings:
|
|
- name: base_url
|
|
kind: string
|
|
value: https://content.explore-education-statistics.service.gov.uk/api/v1
|
|
- name: datasets
|
|
kind: array
|
|
description: List of EES dataset configs to extract
|
|
|
|
- name: tap-uk-ofsted
|
|
namespace: uk_ofsted
|
|
pip_url: ./plugins/extractors/tap-uk-ofsted
|
|
executable: tap-uk-ofsted
|
|
capabilities:
|
|
- catalog
|
|
- state
|
|
settings:
|
|
- name: mi_url
|
|
kind: string
|
|
description: Ofsted Management Information download URL
|
|
|
|
- name: tap-uk-parent-view
|
|
namespace: uk_parent_view
|
|
pip_url: ./plugins/extractors/tap-uk-parent-view
|
|
executable: tap-uk-parent-view
|
|
capabilities:
|
|
- catalog
|
|
|
|
- name: tap-uk-fbit
|
|
namespace: uk_fbit
|
|
pip_url: ./plugins/extractors/tap-uk-fbit
|
|
executable: tap-uk-fbit
|
|
capabilities:
|
|
- catalog
|
|
- state
|
|
settings:
|
|
- name: base_url
|
|
kind: string
|
|
value: https://financial-benchmarking-and-insights-tool.education.gov.uk/api
|
|
|
|
- name: tap-uk-idaci
|
|
namespace: uk_idaci
|
|
pip_url: ./plugins/extractors/tap-uk-idaci
|
|
executable: tap-uk-idaci
|
|
capabilities:
|
|
- catalog
|
|
|
|
loaders:
|
|
- name: target-postgres
|
|
variant: meltanolabs
|
|
pip_url: meltanolabs-target-postgres
|
|
config:
|
|
host: $PG_HOST
|
|
port: $PG_PORT
|
|
user: $PG_USER
|
|
password: $PG_PASSWORD
|
|
dbname: $PG_DATABASE
|
|
default_target_schema: raw
|
|
|
|
utilities:
|
|
- name: dbt-postgres
|
|
variant: dbt-labs
|
|
pip_url: dbt-postgres~=1.10
|
|
config:
|
|
project_dir: $MELTANO_PROJECT_ROOT/transform
|
|
profiles_dir: $MELTANO_PROJECT_ROOT/transform
|
|
|
|
environments:
|
|
- name: dev
|
|
config:
|
|
plugins:
|
|
loaders:
|
|
- name: target-postgres
|
|
config:
|
|
host: localhost
|
|
port: 5432
|
|
user: postgres
|
|
password: postgres
|
|
dbname: school_compare
|
|
- name: production
|
|
config:
|
|
plugins:
|
|
loaders:
|
|
- name: target-postgres
|
|
config:
|
|
host: ${PG_HOST}
|
|
port: ${PG_PORT}
|
|
user: ${PG_USER}
|
|
password: ${PG_PASSWORD}
|
|
dbname: ${PG_DATABASE}
|