diff --git a/docker-compose.portainer.staging.yml b/docker-compose.portainer.staging.yml index 3bbad6b..c08ae48 100644 --- a/docker-compose.portainer.staging.yml +++ b/docker-compose.portainer.staging.yml @@ -18,7 +18,10 @@ # TYPESENSE_SEARCH_KEY — Typesense search-only key (exposed to frontend) # UNLEASH_URL — http://:4242/api (empty = all flags off) # UNLEASH_API_TOKEN — Unleash *client* token, environment: development -# AIRFLOW_ADMIN_USER — Airflow admin username (password auto-generated, see api-server logs) +# AIRFLOW_ADMIN_USER — Airflow admin username (default: admin) +# AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server +# refuses to start without it, rather than falling +# back to a generated one that changes on restart. # STAGING_DB_IP — macvlan IP for staging Postgres (default 10.0.1.190) # STAGING_FRONTEND_IP — macvlan IP for staging frontend (default 10.0.1.151) @@ -124,7 +127,23 @@ services: airflow-api-server: image: privaterepo.sitaru.org/tudor/school_compare-pipeline:staging container_name: sc_staging_airflow_api - command: airflow api-server --port 8080 + # The simple auth manager generates a random password on first start and + # writes it to a file, so every container restart invalidates the last one. + # Writing the file ourselves from an environment variable makes the login + # deterministic. Airflow does not generate anything when the file exists. + # + # Built with python rather than echo/printf so a password containing quotes, + # backslashes or spaces is escaped correctly by json.dumps. An unset + # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling + # back to a generated password would silently undo the point of this. + command: + - bash + - -c + - | + set -euo pipefail + mkdir -p /opt/airflow + python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))" + exec airflow api-server --port 8080 ports: - "8081:8080" environment: @@ -136,6 +155,8 @@ services: AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-staging-airflow-jwt-secret-key-long-enough-for-sha512" AIRFLOW__API_AUTH__JWT_ISSUER: airflow AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin" + AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json + AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?set AIRFLOW_ADMIN_PASSWORD in the Portainer stack environment} AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs PG_HOST: sc_database PG_PORT: "5432" diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml index e44a9b6..00e085e 100644 --- a/docker-compose.portainer.yml +++ b/docker-compose.portainer.yml @@ -9,7 +9,10 @@ # TYPESENSE_SEARCH_KEY — Typesense search-only key (exposed to frontend) # UNLEASH_URL — http://:4242/api (empty = all flags off) # UNLEASH_API_TOKEN — Unleash *client* token, environment: production -# AIRFLOW_ADMIN_USER — Airflow admin username (password auto-generated, see api-server logs) +# AIRFLOW_ADMIN_USER — Airflow admin username (default: admin) +# AIRFLOW_ADMIN_PASSWORD — Airflow admin password. REQUIRED: the api-server +# refuses to start without it, rather than falling +# back to a generated one that changes on restart. services: @@ -113,7 +116,23 @@ services: airflow-api-server: image: privaterepo.sitaru.org/tudor/school_compare-pipeline:prod container_name: schoolcompare_airflow_api - command: airflow api-server --port 8080 + # The simple auth manager generates a random password on first start and + # writes it to a file, so every container restart invalidates the last one. + # Writing the file ourselves from an environment variable makes the login + # deterministic. Airflow does not generate anything when the file exists. + # + # Built with python rather than echo/printf so a password containing quotes, + # backslashes or spaces is escaped correctly by json.dumps. An unset + # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling + # back to a generated password would silently undo the point of this. + command: + - bash + - -c + - | + set -euo pipefail + mkdir -p /opt/airflow + python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))" + exec airflow api-server --port 8080 ports: - "8080:8080" environment: @@ -125,6 +144,8 @@ services: AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512" AIRFLOW__API_AUTH__JWT_ISSUER: airflow AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin" + AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json + AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?set AIRFLOW_ADMIN_PASSWORD in the Portainer stack environment} AIRFLOW__LOGGING__BASE_LOG_FOLDER: /opt/airflow/logs PG_HOST: sc_database PG_PORT: "5432" diff --git a/docker-compose.yml b/docker-compose.yml index b3fdd41..9ed0b74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,7 +105,23 @@ services: airflow-api-server: image: privaterepo.sitaru.org/tudor/school_compare-pipeline:latest container_name: schoolcompare_airflow_api - command: airflow api-server --port 8080 + # The simple auth manager generates a random password on first start and + # writes it to a file, so every container restart invalidates the last one. + # Writing the file ourselves from an environment variable makes the login + # deterministic. Airflow does not generate anything when the file exists. + # + # Built with python rather than echo/printf so a password containing quotes, + # backslashes or spaces is escaped correctly by json.dumps. An unset + # AIRFLOW_ADMIN_PASSWORD raises KeyError and the container exits: falling + # back to a generated password would silently undo the point of this. + command: + - bash + - -c + - | + set -euo pipefail + mkdir -p /opt/airflow + python -c "import json, os, pathlib; pathlib.Path('/opt/airflow/simple_auth_manager_passwords.json').write_text(json.dumps({os.environ.get('AIRFLOW_ADMIN_USER', 'admin'): os.environ['AIRFLOW_ADMIN_PASSWORD']}))" + exec airflow api-server --port 8080 ports: - "8080:8080" environment: &airflow-env @@ -117,6 +133,8 @@ services: AIRFLOW__API_AUTH__JWT_SECRET: "school-compare-airflow-jwt-secret-key-long-enough-for-sha512" AIRFLOW__API_AUTH__JWT_ISSUER: airflow AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "admin:admin" + AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/simple_auth_manager_passwords.json + AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-admin} PG_HOST: db PG_PORT: "5432" PG_USER: schoolcompare diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index d66c4f6..2cff1b8 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -98,6 +98,12 @@ fail the E2E gate. That's the point: staging absorbs the risk. pr-checks status checks (frontend, backend, builds, ai-review) to pass. 5. **Bootstrap staging data via Airflow** (no prod dump — staging populates itself from source, exercising the pipeline image end-to-end): + - Set `AIRFLOW_ADMIN_PASSWORD` in the stack environment first. The + api-server refuses to start without it. Airflow's simple auth manager + otherwise generates a password on first start and writes it to a file, so + the login changes every time the container restarts; the stack writes that + file itself from this variable instead. `AIRFLOW_ADMIN_USER` defaults to + `admin`. - Open the staging Airflow UI (`http://:8081`) and trigger, in order: `school_data_daily`, `school_data_monthly_ofsted`, then the manual-schedule `school_data_annual_ees` and `school_data_annual_idaci`.