diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 55b44edcd..3b287b06a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -360,22 +360,24 @@ jobs:
 
       - name: Gather data
         run: |
-          cargo run --bin collector bench_local `which rustc` --include syn --id version1 --db postgresql://postgres:postgres@127.0.0.1:5432/postgres
-          cargo run --bin collector bench_local `rustup +nightly which rustc` --include syn --id version2 --db postgresql://postgres:postgres@127.0.0.1:5432/postgres
+          cargo run --bin collector bench_local `which rustc` --include syn --id version1 \
+            --self-profile \
+            --db postgresql://postgres:postgres@127.0.0.1:5432/postgres
+          cargo run --bin collector bench_local `rustup +nightly which rustc` --include syn --id version2 \
+            --self-profile \
+            --db postgresql://postgres:postgres@127.0.0.1:5432/postgres
 
       - name: Build site
         run: cargo build --bin site
 
+      - name: Setup Python
+        uses: actions/setup-python@v5
+
+      - name: Install Python dependencies
+        run: python3 -m pip install msgpack requests
+
       # Check that data from the /get endpoint can be successfully queried.
       - name: Query compare page data
         run: |
           DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres cargo run --bin site &
-          curl http://localhost:2346/perf/get \
-            -H 'Content-Type:application/json' \
-            -d '{"start": "version1", "end": "version2", "stat": "instructions:u" }' \
-            --output out.msgpack \
-            --retry-connrefused \
-            --connect-timeout 5 \
-            --max-time 10 \
-            --retry 3 \
-            --retry-delay 5
+          python3 ci/check-site.py version1 version2
diff --git a/ci/check-site.py b/ci/check-site.py
new file mode 100644
index 000000000..13f274aaf
--- /dev/null
+++ b/ci/check-site.py
@@ -0,0 +1,50 @@
+"""
+Checks that the perf site running locally returns non-empty data for a set of artifacts.
+"""
+import sys
+import time
+
+import msgpack
+import requests
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("Usage: python3 check-site.py <version1> <version2>")
+        exit(1)
+    version1 = sys.argv[1]
+    version2 = sys.argv[2]
+
+    # Wait for the site to start
+    while True:
+        try:
+            response = requests.post("http://localhost:2346/perf/get", json={
+                "start": version1,
+                "end": version2,
+                "stat": "instructions:u"
+            })
+            if response.content != b"no data yet, please wait":
+                break
+        except BaseException as e:
+            print(e)
+
+        print(f"Site not online yet, waiting")
+        time.sleep(1)
+
+    # instructions:u is not available on CI, so check at least wall time and binary size
+    stats = ("wall-time", "size:linked_artifact")
+    for stat in stats:
+        print(f"Checking {stat}")
+        response = requests.post("http://localhost:2346/perf/get", json={
+            "start": version1,
+            "end": version2,
+            "stat": stat
+        })
+        if response.status_code != 200:
+            raise Exception(f"Failure {response.status_code}: {response.content}")
+        payload = msgpack.unpackb(response.content)
+        print(payload)
+        for artifact_id in ("a", "b"):
+            artifact = payload[artifact_id]
+            assert artifact["component_sizes"].get("librustc_driver", 0) > 0
+        comparisons = payload["compile_comparisons"]
+        assert len(comparisons) > 0
diff --git a/collector/src/compile/execute/bencher.rs b/collector/src/compile/execute/bencher.rs
index fd9009f61..aa98e75a5 100644
--- a/collector/src/compile/execute/bencher.rs
+++ b/collector/src/compile/execute/bencher.rs
@@ -184,9 +184,6 @@ impl<'a> Processor for BenchProcessor<'a> {
                     let version = get_rustc_perf_commit();
                     let collection = self.conn.collection_id(&version).await;
 
-                    // If the gathered metrics were produced with self profile enabled, then they
-                    // are not realistic. Do not store the metrics into the DB for self-profile
-                    // runs to avoid unnecessary DB storage.
                     if let Some(files) = res.2 {
                         self.self_profiles.push(RecordedSelfProfile {
                             collection,
@@ -194,11 +191,17 @@ impl<'a> Processor for BenchProcessor<'a> {
                             profile,
                             files,
                         });
-                    } else {
-                        self.insert_stats(collection, scenario, profile, data.backend, res.0)
-                            .await;
+
+                        // If the gathered metrics were produced with self profile enabled, then they
+                        // are not realistic. Do not store the metrics that are affected by
+                        // self-profiling into the DB for self-profile runs to avoid unnecessary
+                        // DB storage.
+                        res.0.stats.retain(|key, _| key.starts_with("size:"));
                     }
 
+                    self.insert_stats(collection, scenario, profile, data.backend, res.0)
+                        .await;
+
                     Ok(Retry::No)
                 }
                 Err(DeserializeStatError::NoOutput(output)) => {