school_compare/pipeline/meltano.yml

project_id: school-compare-pipeline
default_environment: production

plugins:
  extractors:
    - name: tap-uk-gias
      namespace: uk_gias
      pip_url: ./plugins/extractors/tap-uk-gias
      executable: tap-uk-gias
      settings:
        - name: download_url
          kind: string
          description: GIAS bulk CSV download URL

    - name: tap-uk-ees
      namespace: uk_ees
      pip_url: ./plugins/extractors/tap-uk-ees
      executable: tap-uk-ees
      settings:
        - name: base_url
          kind: string
          value: https://content.explore-education-statistics.service.gov.uk/api/v1
        - name: datasets
          kind: array
          description: List of EES dataset configs to extract
        - name: legacy_ks2_urls
          kind: object
          description: "Year code → URL mapping for legacy KS2 CSVs"
        - name: legacy_ks4_urls
          kind: object
          description: "Year code → URL mapping for legacy KS4 ZIPs (england_ks4final.csv inside)"
      config:
        legacy_ks2_urls:
          "201516": "http://10.0.1.224:8081/filebrowser/api/public/dl/iaoSkg1v?inline=true"
          "201617": "http://10.0.1.224:8081/filebrowser/api/public/dl/bqCMUcIH?inline=true"
          "201718": "http://10.0.1.224:8081/filebrowser/api/public/dl/0L61fE_a?inline=true"
          "201819": "http://10.0.1.224:8081/filebrowser/api/public/dl/XJGJ5lG1?inline=true"
        legacy_ks4_urls:
          "201516": "http://10.0.1.224:8081/filebrowser/api/public/dl/iaoSkg1v?inline=true"
          "201617": "http://10.0.1.224:8081/filebrowser/api/public/dl/bqCMUcIH?inline=true"
          "201718": "http://10.0.1.224:8081/filebrowser/api/public/dl/0L61fE_a?inline=true"
          "201819": "http://10.0.1.224:8081/filebrowser/api/public/dl/XJGJ5lG1?inline=true"

    - name: tap-uk-ees-destinations
      namespace: uk_ees_destinations
      pip_url: ./plugins/extractors/tap-uk-ees-destinations
      executable: tap-uk-ees-destinations
      settings: []

    - name: tap-uk-ofsted
      namespace: uk_ofsted
      pip_url: ./plugins/extractors/tap-uk-ofsted
      executable: tap-uk-ofsted
      settings:
        - name: mi_url
          kind: string
          description: Ofsted Management Information download URL
        - name: independent_mi_url
          kind: string
          description: Ofsted Independent Schools Management Information download URL

    - name: tap-uk-fbit
      namespace: uk_fbit
      pip_url: ./plugins/extractors/tap-uk-fbit
      executable: tap-uk-fbit
      settings:
        - name: base_url
          kind: string
          value: https://financial-benchmarking-and-insights-tool.education.gov.uk/api

    - name: tap-uk-idaci
      namespace: uk_idaci
      pip_url: ./plugins/extractors/tap-uk-idaci
      executable: tap-uk-idaci

    # Last distance offered — admission cut-offs collected from LA
    # publications. There is no national source for this, so the URL points at
    # the collected CSV rather than at a government endpoint (same pattern as
    # the legacy KS2/KS4 URLs above).
    - name: tap-uk-school-distance
      namespace: uk_school_distance
      pip_url: ./plugins/extractors/tap-uk-school-distance
      executable: tap-uk-school-distance
      settings:
        - name: download_url
          kind: string
          description: URL of the collected last-distance-offered CSV
      config:
        download_url: http://10.0.1.224:8081/filebrowser/api/public/dl/UU_oEqVr?inline=true

  loaders:
    - name: target-postgres
      variant: meltanolabs
      pip_url: meltanolabs-target-postgres
      config:
        host: $PG_HOST
        port: $PG_PORT
        user: $PG_USER
        password: $PG_PASSWORD
        database: $PG_DATABASE
        default_target_schema: raw

  utilities:
    - name: dbt-postgres
      variant: dbt-labs
      pip_url: dbt-postgres~=1.10
      config:
        project_dir: $MELTANO_PROJECT_ROOT/transform
        profiles_dir: $MELTANO_PROJECT_ROOT/transform

environments:
  - name: dev
    config:
      plugins:
        loaders:
          - name: target-postgres
            config:
              host: localhost
              port: 5432
              user: postgres
              password: postgres
              database: school_compare
  - name: production
    config:
      plugins:
        loaders:
          - name: target-postgres
            config:
              host: ${PG_HOST}
              port: ${PG_PORT}
              user: ${PG_USER}
              password: ${PG_PASSWORD}
              database: ${PG_DATABASE}