fix(ci): make a failed release check say what it actually saw
PR Checks / Frontend Typecheck + Tests (pull_request) Successful in 1m11s
PR Checks / Backend Smoke (pull_request) Successful in 9s
PR Checks / Build Backend (no push) (pull_request) Successful in 17s
PR Checks / Build Frontend (no push) (pull_request) Successful in 1m17s
PR Checks / Build Pipeline (no push) (pull_request) Successful in 11s
PR Checks / AI Code Review (Claude) (pull_request) Successful in 1m4s
PR Checks / Frontend Typecheck + Tests (pull_request) Successful in 1m11s
PR Checks / Backend Smoke (pull_request) Successful in 9s
PR Checks / Build Backend (no push) (pull_request) Successful in 17s
PR Checks / Build Frontend (no push) (pull_request) Successful in 1m17s
PR Checks / Build Pipeline (no push) (pull_request) Successful in 11s
PR Checks / AI Code Review (Claude) (pull_request) Successful in 1m4s
The staging poller swallowed every failure identically, so a run that timed out told us only that the expected release never appeared — not whether the proxy refused us, the endpoint was down, or the containers were still serving an older build. The public staging proxy also answers 403 to urllib's default user agent while the release endpoint is healthy, which looked exactly like a deployment that never arrived. Identify the poller, and report each distinct observation once: HTTP status, connection failure type, invalid JSON, or the release identities actually reported. The timeout error carries the last observation and the identity it wanted. Responses and the base URL stay out of the logs — only validated sha/build_id fields are echoed back. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7ab084dd3a
commit
64ae71d7ab
3 files changed
+118
-6
No files matched your search
+44
-6
@@ -10,6 +10,7 @@ from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
COMPONENTS = ('BACKEND', 'FRONTEND', 'PIPELINE')
|
||||
@@ -88,8 +89,29 @@ def promote(sha):
|
||||
|
||||
|
||||
def matches(payload, sha, build_id):
|
||||
return all(payload.get(component) == {'sha': sha, 'build_id': build_id}
|
||||
for component in ('frontend', 'backend'))
|
||||
return isinstance(payload, dict) and all(
|
||||
payload.get(component) == {'sha': sha, 'build_id': build_id}
|
||||
for component in ('frontend', 'backend'))
|
||||
|
||||
|
||||
def describe_identity(payload):
|
||||
"""Log only release fields, never arbitrary response bodies or secret URLs."""
|
||||
if not isinstance(payload, dict):
|
||||
return 'Invalid release response: expected a JSON object'
|
||||
identities = []
|
||||
for component in ('frontend', 'backend'):
|
||||
identity = payload.get(component)
|
||||
if not isinstance(identity, dict):
|
||||
identities.append(f'{component}=missing or invalid')
|
||||
continue
|
||||
values = []
|
||||
for field, length in (('sha', 40), ('build_id', 32)):
|
||||
value = identity.get(field)
|
||||
valid = isinstance(value, str) and (
|
||||
value == 'development' or re.fullmatch(r'[0-9a-f]{' + str(length) + '}', value))
|
||||
values.append(f'{field}={value if valid else "missing or invalid"}')
|
||||
identities.append(f'{component}: {", ".join(values)}')
|
||||
return 'Release mismatch: ' + '; '.join(identities)
|
||||
|
||||
|
||||
def wait(base_url, sha, build_id, timeout):
|
||||
@@ -97,19 +119,35 @@ def wait(base_url, sha, build_id, timeout):
|
||||
if not re.fullmatch(r'[0-9a-f]{32}', build_id):
|
||||
raise ValueError('Missing expected build identity')
|
||||
deadline = time.monotonic() + timeout
|
||||
last_observation = 'No response received'
|
||||
print(f'Waiting for deployed release {sha} / {build_id}', flush=True)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
req = Request(f'{base_url.rstrip("/")}/release.json?check={time.time_ns()}',
|
||||
headers={'Cache-Control': 'no-cache'})
|
||||
headers={'Cache-Control': 'no-cache',
|
||||
'User-Agent': 'SchoolCompare-Release-Check/1.0',
|
||||
'Accept': 'application/json'})
|
||||
with urlopen(req, timeout=min(10, max(.1, deadline - time.monotonic()))) as response:
|
||||
payload = json.load(response)
|
||||
if matches(payload, sha, build_id):
|
||||
print(f'Verified deployed release {sha} / {build_id}')
|
||||
return
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
observation = describe_identity(payload)
|
||||
except HTTPError as exc:
|
||||
observation = f'Release endpoint returned HTTP {exc.code}'
|
||||
exc.close()
|
||||
except URLError as exc:
|
||||
observation = f'Release endpoint connection failed ({type(exc.reason).__name__})'
|
||||
except OSError as exc:
|
||||
observation = f'Release endpoint request failed ({type(exc).__name__})'
|
||||
except ValueError:
|
||||
observation = 'Release endpoint returned invalid JSON or request configuration'
|
||||
if observation != last_observation:
|
||||
print(observation, flush=True)
|
||||
last_observation = observation
|
||||
time.sleep(min(5, max(0, deadline - time.monotonic())))
|
||||
raise RuntimeError('Deployment did not report the expected frontend/backend release')
|
||||
raise RuntimeError('Deployment did not report the expected frontend/backend release '
|
||||
f'{sha} / {build_id}. Last observation: {last_observation}')
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Reference in new issue
Block a user