backend/app.py

@@ -823,11 +823,11 @@ def _national_averages_payload(df: pd.DataFrame) -> dict:
     /api/compare.
 
     Both series are persisted marts computed at import time: official DfE
-    KS2 figures (fact_ks2_national_averages) and dataset-computed KS4
-    averages (fact_ks4_national_averages) — the API never aggregates the
-    performance dataframe per request. If the KS4 mart hasn't been built
-    yet (deploy lands before the next DAG run), fall back to computing the
-    latest year only — a single-year scan, never the historical loop.
+    KS2 figures (fact_ks2_national_averages) and official DfE KS4 figures
+    (fact_ks4_national_averages) — the API never aggregates the performance
+    dataframe per request. If the KS4 mart hasn't been built yet, the
+    secondary series is empty — never a computed stand-in, because the UI
+    labels these figures as official DfE data.
     """
     if df.empty:
         return {"primary": {}, "secondary": {}}
@@ -867,23 +867,6 @@ def _national_averages_payload(df: pd.DataFrame) -> dict:
     primary_by_year = {r.year: _row_metrics(r, _KS2_NATIONAL_METRICS) for r in ks2_rows}
     secondary_by_year = {r.year: _row_metrics(r, _KS4_NATIONAL_METRICS) for r in ks4_rows}
 
-    if not any(secondary_by_year.values()):
-        # KS4 mart missing/empty: compute the latest year only.
-        df_latest = df[df["year"] == latest_year]
-        sec = (
-            df_latest[df_latest["attainment_8_score"].notna()]
-            if "attainment_8_score" in df_latest.columns
-            else df_latest.iloc[0:0]
-        )
-        vals = {}
-        for col in _KS4_NATIONAL_METRICS:
-            if col in sec.columns:
-                v = sec[col].dropna()
-                if len(v) > 0:
-                    vals[col] = round(float(v.mean()), 2)
-        if vals:
-            secondary_by_year[latest_year] = vals
-
     all_years = sorted(set(primary_by_year) | set(secondary_by_year))
     by_year = [
         {

backend/models.py

@@ -251,7 +251,10 @@ class CensusBenchmark(Base):
 
 
 class Ks4NationalAverage(Base):
-    """Computed national KS4 averages (from our dataset) — one row per year."""
+    """Official DfE KS4 national headline averages — one row per academic year.
+
+    gcse_grade_91_pct has no official national series and is always NULL.
+    """
     __tablename__ = "fact_ks4_national_averages"
     __table_args__ = MARTS
 

backend/tests/test_national_averages_marts.py

@@ -1,7 +1,7 @@
 """_national_averages_payload reads persisted marts (computed at import
-time) — it must never loop the dataframe per year. The only dataframe work
-allowed is the single-latest-year KS4 fallback for the window between a
-deploy and the next DAG run."""
+time) — it must never aggregate the dataframe. Both marts hold OFFICIAL
+DfE figures, so a missing KS4 mart yields an empty secondary series —
+never a computed stand-in the UI would mislabel as official."""
 
 import numpy as np
 import pandas as pd
@@ -84,10 +84,11 @@ def test_ks4_averages_come_from_the_mart_not_the_dataframe(payload):
     assert body["by_year"][-1]["secondary"]["progress_8_score"] == -0.02
 
 
-def test_missing_ks4_mart_falls_back_to_latest_year_only(payload):
+def test_ks4_secondary_empty_when_mart_missing(payload):
+    # No computed stand-in: the UI labels national figures as official DfE
+    # data, so an empty mart must yield an empty secondary series.
     body = payload(_Ks4MissingSession)
-    # Fallback computes the latest year from the df: mean(50, 30) = 40.0
-    assert body["secondary"]["attainment_8_score"] == 40.0
-    # ...and only the latest year — no historical KS4 loop
-    ks4_years = [e["year"] for e in body["by_year"] if e["secondary"]]
-    assert ks4_years == [LATEST]
+    assert body["secondary"] == {}
+    assert all(not e["secondary"] for e in body["by_year"])
+    # The KS2 series is unaffected.
+    assert body["primary"]["rwm_expected_pct"] == 62.1

pipeline/plugins/extractors/tap-uk-ees/tap_uk_ees/tap.py

@@ -564,6 +564,74 @@ class EESKs2NationalStream(Stream):
             yield record
 
 
+# ── KS4 National Headlines (national level only — one row per year) ──────────
+# Dataset: "National characteristics summary data" (Key stage 4 performance).
+# Official England state-funded headline measures, 2018/19 → latest.
+# Suppressed values ('z', 'x') → NULL downstream. Progress 8 is legitimately
+# absent in years with no KS2 baseline (e.g. 2024/25) — that is DfE policy,
+# not missing data.
+
+_KS4_NATIONAL_CSV_URL = (
+    "https://explore-education-statistics.service.gov.uk/data-catalogue/"
+    "data-set/1b649e16-01e8-435b-a814-56be2faf9054/csv"
+)
+
+_KS4_NATIONAL_COL_MAP = {
+    "attainment8_average":    "attainment_8_score",
+    "progress8_average":      "progress_8_score",
+    "engmath_94_percent":     "english_maths_standard_pass_pct",
+    "engmath_95_percent":     "english_maths_strong_pass_pct",
+    "ebacc_entering_percent": "ebacc_entry_pct",
+    "ebacc_94_percent":       "ebacc_standard_pass_pct",
+    "ebacc_95_percent":       "ebacc_strong_pass_pct",
+    "ebacc_aps_average":      "ebacc_avg_score",
+}
+
+
+class EESKs4NationalStream(Stream):
+    """National KS4 headline averages — one row per academic year.
+
+    Filters to geographic_level == 'National', establishment_type_group ==
+    'All state-funded', breakdown_topic == 'Total', breakdown == 'Total'
+    so only the England-wide all-pupils row per year is emitted.
+    """
+
+    name = "ees_ks4_national"
+    primary_keys = ["time_period"]
+    replication_key = None
+
+    schema = th.PropertiesList(
+        th.Property("time_period", th.StringType, required=True),
+        *[th.Property(out, th.StringType) for out in _KS4_NATIONAL_COL_MAP.values()],
+    ).to_dict()
+
+    def get_records(self, context):
+        import pandas as pd
+
+        self.logger.info("Downloading KS4 national headlines: %s", _KS4_NATIONAL_CSV_URL)
+        resp = requests.get(_KS4_NATIONAL_CSV_URL, timeout=60)
+        resp.raise_for_status()
+
+        df = pd.read_csv(io.BytesIO(resp.content), dtype=str, keep_default_na=False)
+        df.columns = [c.strip().lower() for c in df.columns]
+
+        for col, want in [
+            ("geographic_level", "national"),
+            ("establishment_type_group", "all state-funded"),
+            ("breakdown_topic", "total"),
+            ("breakdown", "total"),
+        ]:
+            if col in df.columns:
+                df = df[df[col].str.strip().str.lower() == want]
+
+        self.logger.info("Emitting %d national KS4 rows", len(df))
+        for _, row in df.iterrows():
+            record = {"time_period": row.get("time_period", "").strip()}
+            for csv_col, field in _KS4_NATIONAL_COL_MAP.items():
+                record[field] = row.get(csv_col, "").strip()
+            yield record
+
+
 # ── Legacy KS2 (pre-COVID wide format from DfE performance tables) ────────────
 # The DfE "Compare School Performance" site published school-level KS2 CSVs
 # in a wide format (one row per school, ~300 columns).  EES only has school-level
@@ -903,6 +971,7 @@ class TapUKEES(Tap):
             LegacyKS2Stream(self),
             LegacyKS4Stream(self),
             EESKs2NationalStream(self),
+            EESKs4NationalStream(self),
         ]
 
 

pipeline/transform/.user.yml

@@ -0,0 +1 @@
+id: d8efb97e-d188-4585-899c-3ba2e484f763

pipeline/transform/logs/dbt.log

@@ -0,0 +1,294 @@
+22:17:04.566580 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x108c8bcb0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1096f31d0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1096f2e10>]}
+
+
+============================== 22:17:04.568149 | a6b8403f-0ead-494c-bc76-2492aa4c9187 ==============================
+22:17:04.568149 [info ] [MainThread]: Running with dbt=1.11.12
+22:17:04.568347 [debug] [MainThread]: running dbt with arguments {'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'partial_parse': 'True', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'debug': 'False', 'log_format': 'default', 'profiles_dir': '.', 'version_check': 'True', 'write_json': 'True', 'warn_error': 'None', 'invocation_command': 'dbt parse --profiles-dir .', 'static_parser': 'True', 'no_print': 'None', 'use_colors': 'True', 'log_cache_events': 'False', 'indirect_selection': 'eager', 'fail_fast': 'False', 'quiet': 'False', 'send_anonymous_usage_stats': 'True', 'printer_width': '80', 'introspect': 'True', 'empty': 'None', 'target_path': 'None', 'use_experimental_parser': 'False', 'cache_selected_only': 'False'}
+22:17:04.615542 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': 'a6b8403f-0ead-494c-bc76-2492aa4c9187', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x108ded4c0>]}
+22:17:04.631989 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': 'a6b8403f-0ead-494c-bc76-2492aa4c9187', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1097f8800>]}
+22:17:04.632286 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+22:17:04.667716 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+22:17:04.668031 [info ] [MainThread]: Unable to do partial parsing because saved manifest not found. Starting full parse.
+22:17:04.668152 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'partial_parser', 'label': 'a6b8403f-0ead-494c-bc76-2492aa4c9187', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10a4ab8c0>]}
+22:17:05.079216 [warn ] [MainThread]: [WARNING][PropertyMovedToConfigDeprecation]: Deprecated functionality
+Found `severity` as a top-level property of `data_tests.not_null.severity` in
+file `models/marts/_marts_schema.yml`. The `severity` top-level property should
+be moved into the `config` of `data_tests.not_null.severity`.
+22:17:05.079456 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': 'a6b8403f-0ead-494c-bc76-2492aa4c9187', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10aabbb60>]}
+22:17:05.080394 [warn ] [MainThread]: [WARNING][MissingArgumentsPropertyInGenericTestDeprecation]: Deprecated
+functionality
+Found top-level arguments to test `accepted_values` defined on 'dim_school' in
+package 'school_compare' (models/marts/_marts_schema.yml). Arguments to generic
+tests should be nested under the `arguments` property.
+22:17:05.080506 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': 'a6b8403f-0ead-494c-bc76-2492aa4c9187', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1096a4d70>]}
+22:17:05.142077 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': 'a6b8403f-0ead-494c-bc76-2492aa4c9187', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ab519a0>]}
+22:17:05.143319 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+22:17:05.201433 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+22:17:05.202316 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+22:17:05.202527 [warn ] [MainThread]: [WARNING][DeprecationsSummary]: Deprecated functionality
+Summary of encountered deprecations:
+- PropertyMovedToConfigDeprecation: 4 occurrences
+- MissingArgumentsPropertyInGenericTestDeprecation: 7 occurrences
+To see all deprecation instances instead of just the first occurrence of each,
+run command again with the `--show-all-deprecations` flag. You may also need to
+run with `--no-partial-parse` as some deprecations are only encountered during
+parsing.
+22:17:05.202750 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.6586652, "process_in_blocks": "0", "process_kernel_time": 0.111809, "process_mem_max_rss": "135282688", "process_out_blocks": "0", "process_user_time": 0.996675}
+22:17:05.202880 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 22:17:05.202853 after 0.66 seconds
+22:17:05.202995 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10aaed070>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10aaefc50>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10aaed940>]}
+22:17:05.203093 [debug] [MainThread]: Flushing usage events
+22:17:05.735599 [debug] [MainThread]: An error was encountered while trying to flush usage events
+08:22:12.701427 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10c9b8320>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d5f3770>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d9daf00>]}
+
+
+============================== 08:22:12.703695 | 25902cd9-3a92-4695-a273-6c9f7b75e00c ==============================
+08:22:12.703695 [info ] [MainThread]: Running with dbt=1.11.12
+08:22:12.703893 [debug] [MainThread]: running dbt with arguments {'invocation_command': 'dbt parse --profiles-dir .', 'version_check': 'True', 'cache_selected_only': 'False', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'printer_width': '80', 'fail_fast': 'False', 'use_colors': 'True', 'use_experimental_parser': 'False', 'partial_parse': 'True', 'introspect': 'True', 'indirect_selection': 'eager', 'quiet': 'False', 'log_format': 'default', 'write_json': 'True', 'no_print': 'None', 'log_cache_events': 'False', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'send_anonymous_usage_stats': 'True', 'debug': 'False', 'static_parser': 'True', 'empty': 'None', 'target_path': 'None', 'profiles_dir': '.', 'warn_error': 'None'}
+08:22:12.771910 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '25902cd9-3a92-4695-a273-6c9f7b75e00c', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e0d6e40>]}
+08:22:12.788094 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '25902cd9-3a92-4695-a273-6c9f7b75e00c', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e11c7a0>]}
+08:22:12.788548 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+08:22:12.827043 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+08:22:12.890911 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 0 files added, 2 files changed.
+08:22:12.891346 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/marts/_marts_schema.yml
+08:22:12.891484 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/staging/stg_ees_ks4.sql
+08:22:12.988494 [warn ] [MainThread]: [WARNING][MissingArgumentsPropertyInGenericTestDeprecation]: Deprecated
+functionality
+Found top-level arguments to test `accepted_values` defined on
+'fact_ks4_performance' in package 'school_compare'
+(models/marts/_marts_schema.yml). Arguments to generic tests should be nested
+under the `arguments` property.
+08:22:12.988725 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': '25902cd9-3a92-4695-a273-6c9f7b75e00c', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ed83dd0>]}
+08:22:13.042439 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '25902cd9-3a92-4695-a273-6c9f7b75e00c', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ee58050>]}
+08:22:13.043787 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+08:22:13.078667 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+08:22:13.080427 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+08:22:13.080615 [warn ] [MainThread]: [WARNING][DeprecationsSummary]: Deprecated functionality
+Summary of encountered deprecations:
+- MissingArgumentsPropertyInGenericTestDeprecation: 1 occurrence
+To see all deprecation instances instead of just the first occurrence of each,
+run command again with the `--show-all-deprecations` flag. You may also need to
+run with `--no-partial-parse` as some deprecations are only encountered during
+parsing.
+08:22:13.080836 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.40608904, "process_in_blocks": "0", "process_kernel_time": 0.134857, "process_mem_max_rss": "132857856", "process_out_blocks": "0", "process_user_time": 0.702112}
+08:22:13.080968 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 08:22:13.080942 after 0.41 seconds
+08:22:13.081074 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d7a6030>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10eaca450>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10eaca330>]}
+08:22:13.081172 [debug] [MainThread]: Flushing usage events
+08:22:13.497827 [debug] [MainThread]: An error was encountered while trying to flush usage events
+08:22:21.654274 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1081d4920>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1091ff3b0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1091feed0>]}
+
+
+============================== 08:22:21.655842 | 88cd4db0-05ef-4377-87e9-c27e9801255b ==============================
+08:22:21.655842 [info ] [MainThread]: Running with dbt=1.11.12
+08:22:21.656036 [debug] [MainThread]: running dbt with arguments {'no_print': 'None', 'introspect': 'True', 'debug': 'False', 'indirect_selection': 'eager', 'log_cache_events': 'False', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'fail_fast': 'False', 'send_anonymous_usage_stats': 'True', 'profiles_dir': '.', 'printer_width': '80', 'warn_error': 'None', 'partial_parse': 'True', 'static_parser': 'True', 'quiet': 'False', 'write_json': 'True', 'use_experimental_parser': 'False', 'version_check': 'True', 'empty': 'None', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'cache_selected_only': 'False', 'log_format': 'default', 'invocation_command': 'dbt parse --profiles-dir .', 'target_path': 'None', 'use_colors': 'True'}
+08:22:21.706619 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '88cd4db0-05ef-4377-87e9-c27e9801255b', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x106bf7b00>]}
+08:22:21.724467 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '88cd4db0-05ef-4377-87e9-c27e9801255b', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1099b7380>]}
+08:22:21.724916 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+08:22:21.761988 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+08:22:21.811784 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 0 files added, 0 files changed.
+08:22:21.811980 [debug] [MainThread]: Nothing changed, skipping partial parsing.
+08:22:21.812068 [debug] [MainThread]: Partial parsing enabled, no changes found, skipping parsing
+08:22:21.831419 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '88cd4db0-05ef-4377-87e9-c27e9801255b', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10923adb0>]}
+08:22:21.834895 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+08:22:21.872310 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+08:22:21.873270 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+08:22:21.873537 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.24312946, "process_in_blocks": "0", "process_kernel_time": 0.106534, "process_mem_max_rss": "125960192", "process_out_blocks": "0", "process_user_time": 0.617116}
+08:22:21.873695 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 08:22:21.873662 after 0.24 seconds
+08:22:21.873825 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109197680>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1091ff3b0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1091ff140>]}
+08:22:21.873930 [debug] [MainThread]: Flushing usage events
+08:22:22.309014 [debug] [MainThread]: An error was encountered while trying to flush usage events
+08:42:39.581769 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10cc684a0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10dc8f410>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10dc8f020>]}
+
+
+============================== 08:42:39.583995 | 14cee99c-c61b-4745-bfdf-db4edcb73881 ==============================
+08:42:39.583995 [info ] [MainThread]: Running with dbt=1.11.12
+08:42:39.584207 [debug] [MainThread]: running dbt with arguments {'debug': 'False', 'empty': 'None', 'indirect_selection': 'eager', 'fail_fast': 'False', 'invocation_command': 'dbt parse --profiles-dir .', 'version_check': 'True', 'quiet': 'False', 'log_cache_events': 'False', 'write_json': 'True', 'log_format': 'default', 'target_path': 'None', 'static_parser': 'True', 'partial_parse': 'True', 'warn_error': 'None', 'printer_width': '80', 'use_experimental_parser': 'False', 'send_anonymous_usage_stats': 'True', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'cache_selected_only': 'False', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'use_colors': 'True', 'introspect': 'True', 'no_print': 'None', 'profiles_dir': '.'}
+08:42:39.642292 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '14cee99c-c61b-4745-bfdf-db4edcb73881', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10dc63f80>]}
+08:42:39.658889 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '14cee99c-c61b-4745-bfdf-db4edcb73881', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10dc27380>]}
+08:42:39.659445 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+08:42:39.697585 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+08:42:39.759748 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 0 files added, 2 files changed.
+08:42:39.760099 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/staging/stg_legacy_ks2.sql
+08:42:39.760226 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/intermediate/int_ks2_with_lineage.sql
+08:42:39.845972 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '14cee99c-c61b-4745-bfdf-db4edcb73881', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ef6a4e0>]}
+08:42:39.847206 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+08:42:39.903495 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+08:42:39.905863 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+08:42:39.906095 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.3489076, "process_in_blocks": "0", "process_kernel_time": 0.123806, "process_mem_max_rss": "129925120", "process_out_blocks": "0", "process_user_time": 0.678581}
+08:42:39.906230 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 08:42:39.906202 after 0.35 seconds
+08:42:39.906338 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ce3c860>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10f0231d0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10f0235c0>]}
+08:42:39.906440 [debug] [MainThread]: Flushing usage events
+08:42:40.369573 [debug] [MainThread]: An error was encountered while trying to flush usage events
+15:46:41.844088 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10be684a0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10beb34d0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10beb30b0>]}
+
+
+============================== 15:46:41.848473 | 721329d8-fb54-4cf3-9b56-85d8b276dc71 ==============================
+15:46:41.848473 [info ] [MainThread]: Running with dbt=1.11.12
+15:46:41.848910 [debug] [MainThread]: running dbt with arguments {'static_parser': 'True', 'indirect_selection': 'eager', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'target_path': 'None', 'log_format': 'default', 'warn_error': 'None', 'no_print': 'None', 'invocation_command': 'dbt parse --profiles-dir .', 'introspect': 'True', 'profiles_dir': '.', 'write_json': 'True', 'partial_parse': 'True', 'send_anonymous_usage_stats': 'True', 'printer_width': '80', 'empty': 'None', 'use_experimental_parser': 'False', 'log_cache_events': 'False', 'version_check': 'True', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'use_colors': 'True', 'debug': 'False', 'fail_fast': 'False', 'cache_selected_only': 'False', 'quiet': 'False'}
+15:46:41.980957 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '721329d8-fb54-4cf3-9b56-85d8b276dc71', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bf721e0>]}
+15:46:42.016972 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '721329d8-fb54-4cf3-9b56-85d8b276dc71', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10beb3710>]}
+15:46:42.018032 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+15:46:42.095204 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+15:46:42.218385 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 0 files added, 1 files changed.
+15:46:42.219081 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/marts/fact_performance.sql
+15:46:42.526466 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '721329d8-fb54-4cf3-9b56-85d8b276dc71', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d283710>]}
+15:46:42.529480 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+15:46:42.606448 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+15:46:42.608498 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+15:46:42.609039 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.81911224, "process_in_blocks": "0", "process_kernel_time": 0.299951, "process_mem_max_rss": "132186112", "process_out_blocks": "0", "process_user_time": 1.506381}
+15:46:42.609313 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 15:46:42.609252 after 0.82 seconds
+15:46:42.609523 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bc54590>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bc21760>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d1659a0>]}
+15:46:42.609727 [debug] [MainThread]: Flushing usage events
+15:46:43.146552 [debug] [MainThread]: An error was encountered while trying to flush usage events
+18:48:43.630692 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bb5c830>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bba7440>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bba7050>]}
+
+
+============================== 18:48:43.634897 | 6f3c22f1-ca3a-438c-b33b-0ec6cd226f50 ==============================
+18:48:43.634897 [info ] [MainThread]: Running with dbt=1.11.12
+18:48:43.635313 [debug] [MainThread]: running dbt with arguments {'profiles_dir': '.', 'log_format': 'default', 'use_experimental_parser': 'False', 'fail_fast': 'False', 'quiet': 'False', 'use_colors': 'True', 'partial_parse': 'True', 'introspect': 'True', 'printer_width': '80', 'log_cache_events': 'False', 'warn_error': 'None', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'version_check': 'True', 'no_print': 'None', 'indirect_selection': 'eager', 'debug': 'False', 'send_anonymous_usage_stats': 'True', 'static_parser': 'True', 'cache_selected_only': 'False', 'write_json': 'True', 'invocation_command': 'dbt parse --profiles-dir .', 'target_path': 'None', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'empty': 'None'}
+18:48:43.757089 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '6f3c22f1-ca3a-438c-b33b-0ec6cd226f50', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bbe1ca0>]}
+18:48:43.792618 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '6f3c22f1-ca3a-438c-b33b-0ec6cd226f50', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10bc83380>]}
+18:48:43.793544 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+18:48:43.871751 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+18:48:43.998163 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 0 files added, 0 files changed.
+18:48:43.998480 [debug] [MainThread]: Nothing changed, skipping partial parsing.
+18:48:43.998646 [debug] [MainThread]: Partial parsing enabled, no changes found, skipping parsing
+18:48:44.038143 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '6f3c22f1-ca3a-438c-b33b-0ec6cd226f50', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10b491490>]}
+18:48:44.040323 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+18:48:44.117414 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+18:48:44.120049 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+18:48:44.120567 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.54391706, "process_in_blocks": "0", "process_kernel_time": 0.253091, "process_mem_max_rss": "125927424", "process_out_blocks": "0", "process_user_time": 1.258876}
+18:48:44.120854 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 18:48:44.120790 after 0.54 seconds
+18:48:44.121082 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x103c273e0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10cebe1b0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10cebe3c0>]}
+18:48:44.121285 [debug] [MainThread]: Flushing usage events
+18:48:44.644908 [debug] [MainThread]: An error was encountered while trying to flush usage events
+13:02:43.139124 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10c5bc4d0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d5e3290>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d5e2ea0>]}
+
+
+============================== 13:02:43.144856 | efc5f728-37e4-4fa6-95c3-c1f95df270dc ==============================
+13:02:43.144856 [info ] [MainThread]: Running with dbt=1.11.12
+13:02:43.145618 [debug] [MainThread]: running dbt with arguments {'profiles_dir': '.', 'invocation_command': 'dbt parse --profiles-dir .', 'version_check': 'True', 'no_print': 'None', 'cache_selected_only': 'False', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'send_anonymous_usage_stats': 'True', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'quiet': 'False', 'use_experimental_parser': 'False', 'indirect_selection': 'eager', 'warn_error': 'None', 'introspect': 'True', 'target_path': 'None', 'static_parser': 'True', 'printer_width': '80', 'use_colors': 'True', 'write_json': 'True', 'debug': 'False', 'empty': 'None', 'fail_fast': 'False', 'log_format': 'default', 'partial_parse': 'True', 'log_cache_events': 'False'}
+13:02:43.349884 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': 'efc5f728-37e4-4fa6-95c3-c1f95df270dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d6f2a50>]}
+13:02:43.403392 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': 'efc5f728-37e4-4fa6-95c3-c1f95df270dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10c879b50>]}
+13:02:43.406778 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+13:02:43.519932 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+13:02:43.686225 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 1 files added, 1 files changed.
+13:02:43.687063 [debug] [MainThread]: Partial parsing: added file: school_compare://models/marts/fact_ks4_national_averages.sql
+13:02:43.687687 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/marts/_marts_schema.yml
+13:02:44.106801 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': 'efc5f728-37e4-4fa6-95c3-c1f95df270dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e973500>]}
+13:02:44.110827 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+13:02:44.211152 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+13:02:44.213866 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+13:02:44.214469 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 1.1430293, "process_in_blocks": "0", "process_kernel_time": 0.439532, "process_mem_max_rss": "132464640", "process_out_blocks": "0", "process_user_time": 1.983706}
+13:02:44.214778 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 13:02:44.214714 after 1.14 seconds
+13:02:44.215029 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e33ae70>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e33a7b0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e33ac00>]}
+13:02:44.215286 [debug] [MainThread]: Flushing usage events
+13:02:44.594244 [debug] [MainThread]: An error was encountered while trying to flush usage events
+14:44:31.378170 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10cb13c50>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e21b2c0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e21aed0>]}
+
+
+============================== 14:44:31.409964 | 944d7fc7-e4b3-4553-bb9c-7709a72e5adb ==============================
+14:44:31.409964 [info ] [MainThread]: Running with dbt=1.11.12
+14:44:31.410683 [debug] [MainThread]: running dbt with arguments {'empty': 'None', 'no_print': 'None', 'warn_error': 'None', 'log_cache_events': 'False', 'use_colors': 'True', 'send_anonymous_usage_stats': 'True', 'target_path': 'None', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'version_check': 'True', 'profiles_dir': '.', 'log_format': 'default', 'cache_selected_only': 'False', 'introspect': 'True', 'use_experimental_parser': 'False', 'debug': 'False', 'printer_width': '80', 'static_parser': 'True', 'partial_parse': 'True', 'fail_fast': 'False', 'write_json': 'True', 'indirect_selection': 'eager', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'quiet': 'False', 'invocation_command': 'dbt parse --profiles-dir .'}
+14:44:33.887704 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '944d7fc7-e4b3-4553-bb9c-7709a72e5adb', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10e5212e0>]}
+14:44:33.937929 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '944d7fc7-e4b3-4553-bb9c-7709a72e5adb', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10d9ab380>]}
+14:44:33.939906 [info ] [MainThread]: Registered adapter: postgres=1.10.2
+14:44:34.042035 [debug] [MainThread]: checksum: f368f5348cf42803b94ef7fb8257ecf3bf9a9a16dd9c8c51fe5172f2dbeb457f, vars: {}, profile: , target: , version: 1.11.12
+14:44:34.197977 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 1 files added, 4 files changed.
+14:44:34.198649 [debug] [MainThread]: Partial parsing: added file: dbt://macros/adapters/runtime.sql
+14:44:34.198997 [debug] [MainThread]: Partial parsing: updated file: dbt://macros/get_custom_name/get_custom_database.sql
+14:44:34.199285 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/staging/stg_ofsted_inspections.sql
+14:44:34.199603 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/marts/fact_ofsted_inspection.sql
+14:44:34.199873 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/intermediate/int_ofsted_latest.sql
+14:44:34.565891 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '944d7fc7-e4b3-4553-bb9c-7709a72e5adb', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10f683f20>]}
+14:44:34.570071 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+14:44:34.729135 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+14:44:34.732116 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+14:44:34.732862 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 3.4214325, "process_in_blocks": "0", "process_kernel_time": 0.845861, "process_mem_max_rss": "154664960", "process_out_blocks": "0", "process_user_time": 3.50279}
+14:44:34.733331 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 14:44:34.733239 after 3.42 seconds
+14:44:34.733686 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1060ef3e0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10f62da30>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10f62cdd0>]}
+14:44:34.733997 [debug] [MainThread]: Flushing usage events
+14:44:35.152478 [debug] [MainThread]: An error was encountered while trying to flush usage events
+19:03:53.808379 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x107984c20>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109710e30>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109b41bb0>]}
+
+
+============================== 19:03:53.821579 | 716689a2-6ae9-48c1-b309-3b58930321dc ==============================
+19:03:53.821579 [info ] [MainThread]: Running with dbt=1.12.0
+19:03:53.821807 [debug] [MainThread]: running dbt with arguments {'target_path': 'None', 'invocation_command': 'dbt parse --profiles-dir .', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'hints_enabled': 'True', 'quiet': 'False', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'version_check': 'True', 'printer_width': '80', 'fail_fast': 'False', 'introspect': 'True', 'warn_error': 'None', 'cache_selected_only': 'False', 'indirect_selection': 'eager', 'maximum_seed_size_mib': '1', 'profiles_dir': '.', 'log_cache_events': 'False', 'write_json': 'True', 'no_print': 'None', 'use_colors': 'True', 'debug': 'False', 'static_parser': 'True', 'empty': 'None', 'send_anonymous_usage_stats': 'True', 'log_format': 'default', 'partial_parse': 'True', 'use_experimental_parser': 'False'}
+19:03:55.079180 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': '716689a2-6ae9-48c1-b309-3b58930321dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109b76300>]}
+19:03:55.098573 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': '716689a2-6ae9-48c1-b309-3b58930321dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109b51700>]}
+19:03:55.099437 [info ] [MainThread]: Registered adapter: postgres=1.11.0
+19:03:55.140959 [debug] [MainThread]: checksum: 67b34be33115abb132a386d054767a080c4d76fc101bebe6b4f686156fa47e48, vars: {}, profile: school_compare, target: dev, version: 1.12.0
+19:03:55.184556 [info ] [MainThread]: Unable to do partial parsing because of a version mismatch
+19:03:55.184766 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'partial_parser', 'label': '716689a2-6ae9-48c1-b309-3b58930321dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109dbabd0>]}
+19:03:55.675944 [warn ] [MainThread]: [WARNING][PropertyMovedToConfigDeprecation]: Deprecated functionality
+Found `severity` as a top-level property of `data_tests.not_null.severity` in
+file `models/marts/_marts_schema.yml`. The `severity` top-level property should
+be moved into the `config` of `data_tests.not_null.severity`.
+19:03:55.676206 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': '716689a2-6ae9-48c1-b309-3b58930321dc', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109f5b110>]}
+19:03:55.677401 [warn ] [MainThread]: [WARNING][MissingArgumentsPropertyInGenericTestDeprecation]: Deprecated
+functionality
+Found top-level arguments to test `accepted_values` defined on 'dim_school' in
+package 'school_compare' (models/marts/_marts_schema.yml). Arguments to generic
+tests should be nested under the `arguments` property.
+19:03:55.677534 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': '716689a2-6ae9-48c1-b309-3b58930321dc', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10b3f3bf0>]}
+19:03:55.748942 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': '716689a2-6ae9-48c1-b309-3b58930321dc', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10b38a150>]}
+19:03:55.750611 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+19:03:55.820228 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+19:03:55.822037 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+19:03:55.840087 [debug] [MainThread]: Wrote artifact OsiDocument to /Users/tudor/projects/school_compare/pipeline/transform/target/osi_document.json
+19:03:55.840391 [warn ] [MainThread]: [WARNING][DeprecationsSummary]: Deprecated functionality
+Summary of encountered deprecations:
+- PropertyMovedToConfigDeprecation: 4 occurrences
+- MissingArgumentsPropertyInGenericTestDeprecation: 7 occurrences
+To see all deprecation instances instead of just the first occurrence of each,
+run command again with the `--show-all-deprecations` flag. You may also need to
+run with `--no-partial-parse` as some deprecations are only encountered during
+parsing.
+19:03:55.840646 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 2.064521, "process_in_blocks": "0", "process_kernel_time": 0.314306, "process_mem_max_rss": "181239808", "process_out_blocks": "0", "process_user_time": 1.929906}
+19:03:55.840808 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 19:03:55.840776 after 2.06 seconds
+19:03:55.840941 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10b542db0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109dddd60>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10b625d00>]}
+19:03:55.841067 [debug] [MainThread]: Flushing usage events
+19:03:56.468885 [debug] [MainThread]: An error was encountered while trying to flush usage events
+19:07:36.556151 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1082e6f00>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1097367b0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109736090>]}
+
+
+============================== 19:07:36.558168 | eb0bb5d4-b111-46cf-9b81-5e2f74744337 ==============================
+19:07:36.558168 [info ] [MainThread]: Running with dbt=1.12.0
+19:07:36.558368 [debug] [MainThread]: running dbt with arguments {'send_anonymous_usage_stats': 'True', 'version_check': 'True', 'profiles_dir': '.', 'use_experimental_parser': 'False', 'empty': 'None', 'fail_fast': 'False', 'target_path': 'None', 'log_format': 'default', 'log_path': '/Users/tudor/projects/school_compare/pipeline/transform/logs', 'printer_width': '80', 'maximum_seed_size_mib': '1', 'indirect_selection': 'eager', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'invocation_command': 'dbt parse --profiles-dir .', 'no_print': 'None', 'log_cache_events': 'False', 'debug': 'False', 'cache_selected_only': 'False', 'static_parser': 'True', 'quiet': 'False', 'partial_parse': 'True', 'hints_enabled': 'True', 'introspect': 'True', 'warn_error': 'None', 'write_json': 'True', 'use_colors': 'True'}
+19:07:36.618364 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'project_id', 'label': 'eb0bb5d4-b111-46cf-9b81-5e2f74744337', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10a47a5d0>]}
+19:07:36.636294 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'adapter_info', 'label': 'eb0bb5d4-b111-46cf-9b81-5e2f74744337', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x109761a60>]}
+19:07:36.636932 [info ] [MainThread]: Registered adapter: postgres=1.11.0
+19:07:36.677947 [debug] [MainThread]: checksum: 67b34be33115abb132a386d054767a080c4d76fc101bebe6b4f686156fa47e48, vars: {}, profile: school_compare, target: dev, version: 1.12.0
+19:07:36.747220 [debug] [MainThread]: Partial parsing enabled: 0 files deleted, 1 files added, 2 files changed.
+19:07:36.747561 [debug] [MainThread]: Partial parsing: added file: school_compare://models/staging/stg_ees_ks4_national.sql
+19:07:36.748386 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/staging/_stg_sources.yml
+19:07:36.748507 [debug] [MainThread]: Partial parsing: updated file: school_compare://models/marts/fact_ks4_national_averages.sql
+19:07:36.981758 [warn ] [MainThread]: [WARNING][MissingArgumentsPropertyInGenericTestDeprecation]: Deprecated
+functionality
+Found top-level arguments to test `relationships` defined on 'dim_location' in
+package 'school_compare' (models/marts/_marts_schema.yml). Arguments to generic
+tests should be nested under the `arguments` property.
+19:07:36.982033 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': 'eb0bb5d4-b111-46cf-9b81-5e2f74744337', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ac6baa0>]}
+19:07:36.989319 [warn ] [MainThread]: [WARNING][PropertyMovedToConfigDeprecation]: Deprecated functionality
+Found `severity` as a top-level property of `data_tests.not_null.severity` in
+file `models/marts/_marts_schema.yml`. The `severity` top-level property should
+be moved into the `config` of `data_tests.not_null.severity`.
+19:07:36.989498 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'deprecation', 'label': 'eb0bb5d4-b111-46cf-9b81-5e2f74744337', 'property_': 'warn', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10acdb770>]}
+19:07:37.071622 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'load_project', 'label': 'eb0bb5d4-b111-46cf-9b81-5e2f74744337', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ad0e810>]}
+19:07:37.072886 [info ] [MainThread]: Performance info: /Users/tudor/projects/school_compare/pipeline/transform/target/perf_info.json
+19:07:37.113602 [debug] [MainThread]: Wrote artifact WritableManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/manifest.json
+19:07:37.114788 [debug] [MainThread]: Wrote artifact SemanticManifest to /Users/tudor/projects/school_compare/pipeline/transform/target/semantic_manifest.json
+19:07:37.123055 [debug] [MainThread]: Wrote artifact OsiDocument to /Users/tudor/projects/school_compare/pipeline/transform/target/osi_document.json
+19:07:37.123294 [warn ] [MainThread]: [WARNING][DeprecationsSummary]: Deprecated functionality
+Summary of encountered deprecations:
+- MissingArgumentsPropertyInGenericTestDeprecation: 7 occurrences
+- PropertyMovedToConfigDeprecation: 4 occurrences
+To see all deprecation instances instead of just the first occurrence of each,
+run command again with the `--show-all-deprecations` flag. You may also need to
+run with `--no-partial-parse` as some deprecations are only encountered during
+parsing.
+19:07:37.123506 [debug] [MainThread]: Resource report: {"command_name": "parse", "command_success": true, "command_wall_clock_time": 0.59700793, "process_in_blocks": "0", "process_kernel_time": 0.129478, "process_mem_max_rss": "146587648", "process_out_blocks": "0", "process_user_time": 0.936024}
+19:07:37.123650 [debug] [MainThread]: Command `python -m dbt.cli.main parse` succeeded at 19:07:37.123622 after 0.60 seconds
+19:07:37.123768 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ae07c20>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10ae367e0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x1096a3140>]}
+19:07:37.123876 [debug] [MainThread]: Flushing usage events
+19:07:37.533590 [debug] [MainThread]: An error was encountered while trying to flush usage events

pipeline/transform/models/marts/fact_ks4_national_averages.sql

@@ -1,25 +1,22 @@
 {{ config(materialized='table') }}
 
--- Mart: Computed national KS4 averages — one row per academic year.
--- Unlike fact_ks2_national_averages (official DfE figures), DfE publishes no
--- KS4 national-headline dataset we ingest yet, so these are means computed
--- across the state schools in our dataset. Computed once at build time so the
--- API never has to aggregate the full performance table per request.
--- Semantics match the API's previous per-request computation: rows where
--- attainment_8_score is non-null; per-column means ignore NULLs.
+-- Mart: OFFICIAL DfE KS4 national headline averages — one row per academic
+-- year (England, state-funded, all pupils), from the EES national dataset.
+-- Replaces the previous unweighted school-level means, which were 7–15
+-- points off every headline measure and produced an arithmetically
+-- impossible national Progress 8. gcse_grade_91_pct has no official
+-- national series and is NULL (schema kept for the API model).
 
 select
     year,
-    round(avg(attainment_8_score)::numeric, 2)               as attainment_8_score,
-    round(avg(progress_8_score)::numeric, 2)                 as progress_8_score,
-    round(avg(english_maths_standard_pass_pct)::numeric, 2)  as english_maths_standard_pass_pct,
-    round(avg(english_maths_strong_pass_pct)::numeric, 2)    as english_maths_strong_pass_pct,
-    round(avg(ebacc_entry_pct)::numeric, 2)                  as ebacc_entry_pct,
-    round(avg(ebacc_standard_pass_pct)::numeric, 2)          as ebacc_standard_pass_pct,
-    round(avg(ebacc_strong_pass_pct)::numeric, 2)            as ebacc_strong_pass_pct,
-    round(avg(ebacc_avg_score)::numeric, 2)                  as ebacc_avg_score,
-    round(avg(gcse_grade_91_pct)::numeric, 2)                as gcse_grade_91_pct
-from {{ ref('fact_ks4_performance') }}
-where attainment_8_score is not null
-group by year
+    attainment_8_score,
+    progress_8_score,
+    english_maths_standard_pass_pct,
+    english_maths_strong_pass_pct,
+    ebacc_entry_pct,
+    ebacc_standard_pass_pct,
+    ebacc_strong_pass_pct,
+    ebacc_avg_score,
+    cast(null as double precision) as gcse_grade_91_pct
+from {{ ref('stg_ees_ks4_national') }}
 order by year

pipeline/transform/models/staging/_stg_sources.yml

@@ -51,6 +51,9 @@ sources:
       - name: ees_ks2_national
         description: KS2 national headline averages from DfE EES data catalogue — one row per academic year
 
+      - name: ees_ks4_national
+        description: Official KS4 national headline averages from DfE EES data catalogue — one row per academic year
+
       # Phonics: no school-level data on EES (only national/LA level)
 
       - name: fbit_finance

pipeline/transform/models/staging/stg_ees_ks4_national.sql

@@ -0,0 +1,20 @@
+{{ config(materialized='table') }}
+
+-- Staging model: official DfE KS4 national headline averages — one row per
+-- academic year (England, all state-funded, all pupils). Source: EES data
+-- catalogue "National characteristics summary data". Suppressed values
+-- ('z', 'x') are coerced to NULL by safe_numeric — Progress 8 is 'z' in
+-- years with no KS2 baseline (e.g. 2024/25): legitimately unpublished.
+
+select
+    cast(trim(time_period) as integer)                        as year,
+    {{ safe_numeric('attainment_8_score') }}                  as attainment_8_score,
+    {{ safe_numeric('progress_8_score') }}                    as progress_8_score,
+    {{ safe_numeric('english_maths_standard_pass_pct') }}     as english_maths_standard_pass_pct,
+    {{ safe_numeric('english_maths_strong_pass_pct') }}       as english_maths_strong_pass_pct,
+    {{ safe_numeric('ebacc_entry_pct') }}                     as ebacc_entry_pct,
+    {{ safe_numeric('ebacc_standard_pass_pct') }}             as ebacc_standard_pass_pct,
+    {{ safe_numeric('ebacc_strong_pass_pct') }}               as ebacc_strong_pass_pct,
+    {{ safe_numeric('ebacc_avg_score') }}                     as ebacc_avg_score
+from {{ source('raw', 'ees_ks4_national') }}
+where time_period ~ '^[0-9]+$'

pipeline/transform/target/osi_document.json

@@ -0,0 +1 @@
+{"version": "0.1.1", "dialects": ["ANSI_SQL"], "semantic_model": [{"name": "semantic_model", "datasets": []}]}

pipeline/transform/target/perf_info.json

@@ -0,0 +1,50 @@
+{
+    "path_count": 181,
+    "parsed_path_count": 37,
+    "static_analysis_path_count": 32,
+    "static_analysis_parsed_path_count": 22,
+    "is_partial_parse_enabled": true,
+    "is_static_analysis_enabled": true,
+    "read_files_elapsed": 0.024260542006231844,
+    "load_macros_elapsed": 0.00022041599731892347,
+    "parse_project_elapsed": 0.2873437089729123,
+    "patch_sources_elapsed": 0.010581917013041675,
+    "process_manifest_elapsed": 0.003397666965611279,
+    "load_all_elapsed": 0.39681549998931587,
+    "projects": [
+        {
+            "project_name": "school_compare",
+            "elapsed": 0.28726037504384294,
+            "parsers": [
+                {
+                    "parser": "model",
+                    "elapsed": 0.1260387499933131,
+                    "parsed_path_count": 32
+                },
+                {
+                    "parser": "test",
+                    "elapsed": 0.03065204102313146,
+                    "parsed_path_count": 3
+                },
+                {
+                    "parser": "test",
+                    "elapsed": 0.1305608749971725,
+                    "parsed_path_count": 2
+                }
+            ],
+            "parsed_path_count": 37
+        },
+        {
+            "project_name": "dbt_postgres",
+            "elapsed": 0,
+            "parsers": [],
+            "parsed_path_count": 0
+        },
+        {
+            "project_name": "dbt",
+            "elapsed": 0,
+            "parsers": [],
+            "parsed_path_count": 0
+        }
+    ]
+}

pipeline/transform/target/semantic_manifest.json

@@ -0,0 +1 @@
+{"semantic_models": [], "metrics": [], "project_configuration": {"time_spine_table_configurations": [], "metadata": null, "dsi_package_version": {"major_version": "0", "minor_version": "211", "patch_version": "0"}, "time_spines": []}, "saved_queries": []}