chore: remove the Ofsted Parent View feature end to end
PR Checks / Frontend Typecheck + Tests (pull_request) Successful in 9m39s
PR Checks / Backend Smoke (pull_request) Successful in 5s
PR Checks / Build Backend (no push) (pull_request) Successful in 17s
PR Checks / Build Frontend (no push) (pull_request) Successful in 41s
PR Checks / Build Pipeline (no push) (pull_request) Successful in 45s
PR Checks / AI Code Review (Claude) (pull_request) Successful in 1m35s
PR Checks / Frontend Typecheck + Tests (pull_request) Successful in 9m39s
PR Checks / Backend Smoke (pull_request) Successful in 5s
PR Checks / Build Backend (no push) (pull_request) Successful in 17s
PR Checks / Build Frontend (no push) (pull_request) Successful in 41s
PR Checks / Build Pipeline (no push) (pull_request) Successful in 45s
PR Checks / AI Code Review (Claude) (pull_request) Successful in 1m35s
Removes the 'What Parents Say' section and all supporting elements: Frontend: - Drop the OfstedParentView type, the parent_view field, the survey section and the 'X% would recommend' callouts in the primary and secondary detail views, the Parents nav item, and the parent-view CSS. Backend: - Remove the FactParentView model, its loading in data_loader, and parent_view from the school-details API response. - Bump SCHEMA_VERSION to 6 and add an idempotent drop step (DROP TABLE IF EXISTS marts.fact_parent_view) to the CLI migration; add scripts/sql/drop_fact_parent_view.sql to apply directly to the dbt-owned marts DBs on staging and prod. Pipeline: - Delete the stg_parent_view + fact_parent_view dbt models and their source/schema entries, the tap-uk-parent-view Meltano extractor, and the monthly Parent View DAG; drop it from the Dockerfile and the staging bootstrap docs. The rest of dbt (which builds every mart the app reads) is untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,18 +0,0 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "tap-uk-parent-view"
|
||||
version = "0.1.0"
|
||||
description = "Singer tap for UK Ofsted Parent View survey data"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"singer-sdk~=0.53",
|
||||
"requests>=2.31",
|
||||
"pandas>=2.0",
|
||||
"openpyxl>=3.1",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
tap-uk-parent-view = "tap_uk_parent_view.tap:TapUKParentView.cli"
|
||||
@@ -1 +0,0 @@
|
||||
"""tap-uk-parent-view: Singer tap for Ofsted Parent View survey data."""
|
||||
@@ -1,151 +0,0 @@
|
||||
"""Parent View Singer tap — extracts survey data from Ofsted Parent View open data portal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from singer_sdk import Stream, Tap
|
||||
from singer_sdk import typing as th
|
||||
|
||||
OPEN_DATA_PAGE = "https://parentview.ofsted.gov.uk/open-data"
|
||||
|
||||
|
||||
def _positive_pct(row: pd.Series, q_col_base: str) -> float | None:
|
||||
"""Sum 'Strongly agree' + 'Agree' percentages for a question."""
|
||||
strongly = row.get(f"{q_col_base} - Strongly agree %") or row.get(f"{q_col_base} - Strongly Agree %")
|
||||
agree = row.get(f"{q_col_base} - Agree %")
|
||||
try:
|
||||
total = 0.0
|
||||
if pd.notna(strongly):
|
||||
total += float(strongly)
|
||||
if pd.notna(agree):
|
||||
total += float(agree)
|
||||
return round(total, 1) if total > 0 else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class ParentViewStream(Stream):
|
||||
"""Stream: Parent View survey responses per school."""
|
||||
|
||||
name = "parent_view"
|
||||
primary_keys = ["urn"]
|
||||
replication_key = None
|
||||
|
||||
schema = th.PropertiesList(
|
||||
th.Property("urn", th.IntegerType, required=True),
|
||||
th.Property("survey_date", th.StringType),
|
||||
th.Property("total_responses", th.IntegerType),
|
||||
th.Property("q_happy_pct", th.NumberType),
|
||||
th.Property("q_safe_pct", th.NumberType),
|
||||
th.Property("q_behaviour_pct", th.NumberType),
|
||||
th.Property("q_bullying_pct", th.NumberType),
|
||||
th.Property("q_communication_pct", th.NumberType),
|
||||
th.Property("q_progress_pct", th.NumberType),
|
||||
th.Property("q_teaching_pct", th.NumberType),
|
||||
th.Property("q_information_pct", th.NumberType),
|
||||
th.Property("q_curriculum_pct", th.NumberType),
|
||||
th.Property("q_future_pct", th.NumberType),
|
||||
th.Property("q_leadership_pct", th.NumberType),
|
||||
th.Property("q_wellbeing_pct", th.NumberType),
|
||||
th.Property("q_recommend_pct", th.NumberType),
|
||||
).to_dict()
|
||||
|
||||
def _discover_download_url(self) -> str:
|
||||
"""Scrape the open data page for the download link."""
|
||||
resp = requests.get(OPEN_DATA_PAGE, timeout=30)
|
||||
resp.raise_for_status()
|
||||
urls = re.findall(r'href="([^"]+\.(?:xlsx|csv|zip))"', resp.text, re.IGNORECASE)
|
||||
if not urls:
|
||||
msg = "No download link found on Parent View open data page"
|
||||
raise RuntimeError(msg)
|
||||
url = urls[0]
|
||||
if not url.startswith("http"):
|
||||
url = "https://parentview.ofsted.gov.uk" + url
|
||||
return url
|
||||
|
||||
def get_records(self, context):
|
||||
url = self._discover_download_url()
|
||||
self.logger.info("Downloading Parent View data: %s", url)
|
||||
|
||||
resp = requests.get(url, timeout=120)
|
||||
resp.raise_for_status()
|
||||
|
||||
if url.endswith(".xlsx"):
|
||||
df = pd.read_excel(io.BytesIO(resp.content))
|
||||
else:
|
||||
df = pd.read_csv(
|
||||
io.BytesIO(resp.content),
|
||||
encoding="latin-1",
|
||||
low_memory=False,
|
||||
)
|
||||
|
||||
# Normalise URN column
|
||||
urn_col = next((c for c in df.columns if c.strip().upper() == "URN"), None)
|
||||
if not urn_col:
|
||||
self.logger.error("URN column not found. Columns: %s", list(df.columns)[:20])
|
||||
return
|
||||
|
||||
df.rename(columns={urn_col: "urn"}, inplace=True)
|
||||
df["urn"] = pd.to_numeric(df["urn"], errors="coerce")
|
||||
df = df.dropna(subset=["urn"])
|
||||
|
||||
# Find total responses column
|
||||
resp_col = next(
|
||||
(c for c in df.columns if "total" in c.lower() and "respon" in c.lower()),
|
||||
None,
|
||||
)
|
||||
|
||||
today = date.today().isoformat()
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
urn = int(row["urn"])
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
total = None
|
||||
if resp_col and pd.notna(row.get(resp_col)):
|
||||
try:
|
||||
total = int(row[resp_col])
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
yield {
|
||||
"urn": urn,
|
||||
"survey_date": today,
|
||||
"total_responses": total,
|
||||
"q_happy_pct": _positive_pct(row, "Q1"),
|
||||
"q_safe_pct": _positive_pct(row, "Q2"),
|
||||
"q_behaviour_pct": _positive_pct(row, "Q3"),
|
||||
"q_bullying_pct": _positive_pct(row, "Q4"),
|
||||
"q_communication_pct": _positive_pct(row, "Q5"),
|
||||
"q_progress_pct": _positive_pct(row, "Q7"),
|
||||
"q_teaching_pct": _positive_pct(row, "Q8"),
|
||||
"q_information_pct": _positive_pct(row, "Q9"),
|
||||
"q_curriculum_pct": _positive_pct(row, "Q10"),
|
||||
"q_future_pct": _positive_pct(row, "Q11"),
|
||||
"q_leadership_pct": _positive_pct(row, "Q12"),
|
||||
"q_wellbeing_pct": _positive_pct(row, "Q13"),
|
||||
"q_recommend_pct": _positive_pct(row, "Q14"),
|
||||
}
|
||||
|
||||
|
||||
class TapUKParentView(Tap):
|
||||
"""Singer tap for UK Ofsted Parent View."""
|
||||
|
||||
name = "tap-uk-parent-view"
|
||||
config_jsonschema = th.PropertiesList(
|
||||
th.Property("download_url", th.StringType, description="Direct URL to Parent View data file"),
|
||||
).to_dict()
|
||||
|
||||
def discover_streams(self):
|
||||
return [ParentViewStream(self)]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
TapUKParentView.cli()
|
||||
Reference in New Issue
Block a user