backend/app/config.py

@@ -5,6 +5,8 @@ class Settings(BaseSettings):
     app_name: str = "MinerOps Console"
     app_env: str = "development"
     app_timezone: str = "America/Winnipeg"
+    minerops_build_commit: str | None = None
+    minerops_build_version: str | None = None
 
     postgres_db: str = "minerops"
     postgres_user: str = "minerops"

backend/app/frontend/static/system-health.css

@@ -0,0 +1,139 @@
+.health-toolbar,
+.health-section-head {
+    display: flex;
+    align-items: flex-start;
+    justify-content: space-between;
+    gap: 16px;
+}
+
+.health-toolbar {
+    margin-bottom: 12px;
+}
+
+.health-toolbar h2,
+.health-toolbar p,
+.health-section h2,
+.health-section h3 {
+    margin-top: 0;
+}
+
+.health-toolbar p {
+    margin-bottom: 0;
+    color: var(--muted-text, #52606d);
+}
+
+.health-notice {
+    border-left: 4px solid #64748b;
+    background: #f1f5f9;
+    margin: 0 0 14px;
+    padding: 10px 12px;
+}
+
+.health-notice[data-state="partial"] {
+    border-left-color: #b45309;
+    background: #fff7ed;
+}
+
+.health-notice[data-state="error"] {
+    border-left-color: #b91c1c;
+    background: #fef2f2;
+}
+
+.health-section {
+    border-top: 1px solid #cbd5e1;
+    padding: 16px 0;
+}
+
+.health-columns,
+.health-integrity-grid,
+.health-facts {
+    display: grid;
+    gap: 14px;
+}
+
+.health-columns {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+
+.health-integrity-grid {
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+}
+
+.health-facts {
+    grid-template-columns: repeat(4, minmax(0, 1fr));
+    margin: 0;
+}
+
+.health-facts-compact {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+
+.health-facts div {
+    min-width: 0;
+}
+
+.health-facts dt {
+    color: #52606d;
+    font-size: 0.78rem;
+    font-weight: 700;
+}
+
+.health-facts dd {
+    font-weight: 700;
+    margin: 3px 0 0;
+    overflow-wrap: anywhere;
+}
+
+.health-links {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 10px;
+}
+
+.health-links a:focus-visible,
+#healthRefresh:focus-visible {
+    outline: 3px solid #0f766e;
+    outline-offset: 3px;
+}
+
+.table-scroll {
+    overflow-x: auto;
+}
+
+.table-scroll table {
+    width: 100%;
+    border-collapse: collapse;
+}
+
+.table-scroll th,
+.table-scroll td {
+    border-bottom: 1px solid #dbe2e8;
+    padding: 7px 8px;
+    text-align: left;
+}
+
+@media (max-width: 900px) {
+    .health-integrity-grid,
+    .health-facts {
+        grid-template-columns: repeat(2, minmax(0, 1fr));
+    }
+}
+
+@media (max-width: 640px) {
+    .health-toolbar,
+    .health-section-head {
+        align-items: stretch;
+        flex-direction: column;
+    }
+
+    .health-columns,
+    .health-integrity-grid,
+    .health-facts,
+    .health-facts-compact {
+        grid-template-columns: 1fr;
+    }
+
+    #healthRefresh {
+        width: 100%;
+    }
+}

backend/app/frontend/static/system-health.js

@@ -0,0 +1,164 @@
+"use strict";
+
+(function () {
+    const notice = document.getElementById("healthNotice");
+    const content = document.getElementById("healthContent");
+    const refresh = document.getElementById("healthRefresh");
+
+    function setText(id, value, fallback) {
+        const element = document.getElementById(id);
+        if (element) {
+            element.textContent = value === null || value === undefined || value === ""
+                ? (fallback || "Unavailable") : String(value);
+        }
+    }
+
+    function number(value) {
+        return typeof value === "number" ? value.toLocaleString() : "Unavailable";
+    }
+
+    function timestamp(value) {
+        if (!value) return "Unavailable";
+        const date = new Date(value);
+        return Number.isNaN(date.valueOf()) ? "Unavailable" : date.toLocaleString();
+    }
+
+    function factualAge(value) {
+        if (!value) return "Unavailable";
+        const observed = new Date(value);
+        if (Number.isNaN(observed.valueOf())) return "Unavailable";
+        const seconds = Math.max(0, Math.floor((Date.now() - observed.valueOf()) / 1000));
+        if (seconds < 60) return seconds + " seconds ago";
+        const minutes = Math.floor(seconds / 60);
+        if (minutes < 60) return minutes + (minutes === 1 ? " minute ago" : " minutes ago");
+        const hours = Math.floor(minutes / 60);
+        if (hours < 24) return hours + (hours === 1 ? " hour ago" : " hours ago");
+        const days = Math.floor(hours / 24);
+        return days + (days === 1 ? " day ago" : " days ago");
+    }
+
+    function replaceFacts(id, facts) {
+        const list = document.getElementById(id);
+        list.replaceChildren();
+        for (const [label, value] of facts) {
+            const row = document.createElement("div");
+            const term = document.createElement("dt");
+            const detail = document.createElement("dd");
+            term.textContent = label;
+            detail.textContent = value;
+            row.append(term, detail);
+            list.append(row);
+        }
+    }
+
+    function render(payload) {
+        const data = payload.data;
+        const app = data.application;
+        const postgres = data.postgresql;
+        const migration = postgres.migration;
+        const telemetry = postgres.telemetry;
+        setText("healthAppStatus", app.status);
+        setText("healthEnvironment", app.environment);
+        setText("healthBuildCommit", app.build_commit);
+        setText("healthBuildVersion", app.build_version);
+        setText("healthServerTime", timestamp(app.server_time));
+        setText("healthMigrationState", migration.state);
+        setText("healthCurrentRevision", migration.current_revision);
+        setText("healthExpectedRevision", migration.expected_revision);
+        setText("healthPostgresqlStatus", postgres.status);
+        setText("healthRedisStatus", data.redis.status);
+        setText("healthSnapshotCount", telemetry ? number(telemetry.snapshot_count) : null);
+        setText("healthOldestObservation", telemetry ? timestamp(telemetry.oldest_observed_at) : null);
+        setText("healthLatestObservation", telemetry ? timestamp(telemetry.latest_observed_at) : null);
+        setText("healthObservationAge", telemetry ? factualAge(telemetry.latest_observed_at) : null);
+
+        const fleet = postgres.fleet;
+        replaceFacts("healthFleetFacts", fleet ? [
+            ["Total miners", number(fleet.total_miners)],
+            ["Without Asset ID", number(fleet.miners_without_asset_id)],
+            ["Unmapped", number(fleet.unmapped_miners)],
+            ["In multiple positions", number(fleet.miners_in_multiple_positions)],
+            ["Duplicate current MAC groups", number(fleet.duplicate_current_mac_groups)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const locations = postgres.locations;
+        replaceFacts("healthLocationFacts", locations ? [
+            ["Sites", number(locations.sites)], ["Rooms", number(locations.rooms)],
+            ["Positions", number(locations.positions)],
+            ["Occupied", number(locations.occupied_positions)],
+            ["Empty", number(locations.empty_positions)],
+            ["Legacy unscoped", number(locations.unscoped_legacy_positions)],
+            ["Expected MAC mismatches", number(locations.expected_mac_mismatches)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const repairs = postgres.repairs;
+        replaceFacts("healthRepairFacts", repairs ? [
+            ["Active modular repairs", number(repairs.active_modular_repairs)],
+            ["Duplicate active groups", number(repairs.duplicate_active_repair_groups)],
+            ["Legacy repair only", number(repairs.legacy_repair_only_miners)],
+            ["Lifecycle repair without active repair", number(repairs.lifecycle_repair_without_active_repair)],
+            ["Active repair lifecycle mismatches", number(repairs.active_repair_lifecycle_mismatches)],
+            ["Waiting placement", number(repairs.waiting_placement_records)],
+            ["Repair events", number(repairs.repair_events)],
+            ["Repair parts", number(repairs.repair_parts)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const retention = postgres.retention;
+        replaceFacts("healthRetentionFacts", retention ? [
+            ["Raw telemetry retention", retention.raw_retention_enabled ? "Enabled" : "Not enabled"],
+            ["Hourly rollups", retention.hourly.table_present ? "Present; manual population" : "Unavailable"],
+            ["Hourly latest bucket", timestamp(retention.hourly.latest_bucket)],
+            ["Daily rollups", retention.daily.table_present ? "Present; manual population" : "Unavailable"],
+            ["Daily latest bucket", timestamp(retention.daily.latest_bucket)],
+            ["Latest recorded run", timestamp(retention.latest_recorded_run_at)],
+            ["Latest run mode", retention.latest_recorded_run_mode || "None recorded"],
+            ["Completeness", "Not guaranteed"]
+        ] : [["Diagnostics", "Unavailable"]]);
+
+        const tableBody = document.getElementById("healthTableCounts");
+        tableBody.replaceChildren();
+        const rows = postgres.table_counts || [];
+        for (const table of rows) {
+            const row = document.createElement("tr");
+            const name = document.createElement("th");
+            const count = document.createElement("td");
+            name.scope = "row";
+            name.textContent = table.name;
+            count.textContent = number(table.row_count);
+            row.append(name, count);
+            tableBody.append(row);
+        }
+        if (!rows.length) {
+            const row = document.createElement("tr");
+            const cell = document.createElement("td");
+            cell.colSpan = 2;
+            cell.textContent = "Table diagnostics unavailable.";
+            row.append(cell);
+            tableBody.append(row);
+        }
+        content.hidden = false;
+        notice.dataset.state = payload.meta.partial ? "partial" : "ready";
+        notice.textContent = payload.meta.partial
+            ? "Diagnostics loaded with unavailable sections: " + payload.meta.unavailable_sections.join(", ") + "."
+            : "Diagnostics refreshed at " + timestamp(payload.meta.generated_at) + ".";
+    }
+
+    async function load() {
+        refresh.disabled = true;
+        notice.dataset.state = "loading";
+        notice.textContent = "Loading diagnostics.";
+        try {
+            const response = await fetch("/api/system/health-summary", {
+                credentials: "same-origin", headers: {"Accept": "application/json"}
+            });
+            const payload = await response.json().catch(function () { return {}; });
+            if (!response.ok) throw new Error("System Health request failed");
+            render(payload);
+        } catch (error) {
+            notice.dataset.state = "error";
+            notice.textContent = "System Health diagnostics could not be loaded.";
+        } finally {
+            refresh.disabled = false;
+        }
+    }
+
+    refresh.addEventListener("click", load);
+    load();
+}());

backend/app/frontend/system_health_view.py

@@ -0,0 +1,54 @@
+"""Semantic content for the Internal Alpha System Health page."""
+
+
+def system_health_content() -> str:
+    return """
+<section class="health-toolbar" aria-labelledby="healthOverviewTitle">
+    <div><h2 id="healthOverviewTitle">Operations diagnostics</h2>
+        <p>Availability, data facts, integrity counts, and maintenance state.</p></div>
+    <button class="primary-button" id="healthRefresh" type="button">Refresh</button>
+</section>
+<p class="health-notice" id="healthNotice" role="status" aria-live="polite">Loading diagnostics.</p>
+<div id="healthContent" hidden>
+    <section class="health-section" aria-labelledby="applicationHealthTitle">
+        <h2 id="applicationHealthTitle">Application</h2>
+        <dl class="health-facts">
+            <div><dt>Application</dt><dd id="healthAppStatus">-</dd></div>
+            <div><dt>Environment</dt><dd id="healthEnvironment">-</dd></div>
+            <div><dt>Build commit</dt><dd id="healthBuildCommit">-</dd></div>
+            <div><dt>Build version</dt><dd id="healthBuildVersion">-</dd></div>
+            <div><dt>Server time</dt><dd id="healthServerTime">-</dd></div>
+            <div><dt>Schema</dt><dd id="healthMigrationState">-</dd></div>
+            <div><dt>Database revision</dt><dd id="healthCurrentRevision">-</dd></div>
+            <div><dt>Expected revision</dt><dd id="healthExpectedRevision">-</dd></div>
+        </dl>
+    </section>
+    <div class="health-columns">
+        <section class="health-section" aria-labelledby="servicesHealthTitle"><h2 id="servicesHealthTitle">Service availability</h2>
+            <dl class="health-facts health-facts-compact"><div><dt>PostgreSQL</dt><dd id="healthPostgresqlStatus">-</dd></div>
+                <div><dt>Redis</dt><dd id="healthRedisStatus">-</dd></div></dl></section>
+        <section class="health-section" aria-labelledby="telemetryHealthTitle"><h2 id="telemetryHealthTitle">Telemetry</h2>
+            <dl class="health-facts health-facts-compact"><div><dt>Raw snapshots</dt><dd id="healthSnapshotCount">-</dd></div>
+                <div><dt>Oldest observation</dt><dd id="healthOldestObservation">-</dd></div>
+                <div><dt>Latest observation</dt><dd id="healthLatestObservation">-</dd></div>
+                <div><dt>Latest age</dt><dd id="healthObservationAge">-</dd></div></dl></section>
+    </div>
+    <section class="health-section" aria-labelledby="integrityHealthTitle">
+        <div class="health-section-head"><h2 id="integrityHealthTitle">Operational integrity facts</h2>
+            <div class="health-links"><a href="/assets/miners">Fleet</a><a href="/monitoring/map">Mine Map</a>
+                <a href="/operations/repairs">Repair Center</a><a href="/monitoring/analytics">Trends</a></div></div>
+        <div class="health-integrity-grid">
+            <section aria-labelledby="fleetIntegrityTitle"><h3 id="fleetIntegrityTitle">Fleet</h3><dl id="healthFleetFacts" class="health-facts health-facts-compact"></dl></section>
+            <section aria-labelledby="locationIntegrityTitle"><h3 id="locationIntegrityTitle">Locations</h3><dl id="healthLocationFacts" class="health-facts health-facts-compact"></dl></section>
+            <section aria-labelledby="repairIntegrityTitle"><h3 id="repairIntegrityTitle">Repairs</h3><dl id="healthRepairFacts" class="health-facts health-facts-compact"></dl></section>
+        </div>
+    </section>
+    <div class="health-columns">
+        <section class="health-section" aria-labelledby="retentionHealthTitle"><h2 id="retentionHealthTitle">Retention and rollups</h2>
+            <dl id="healthRetentionFacts" class="health-facts health-facts-compact"></dl></section>
+        <section class="health-section" aria-labelledby="tablesHealthTitle"><h2 id="tablesHealthTitle">Operational tables</h2>
+            <div class="table-scroll"><table><thead><tr><th scope="col">Table</th><th scope="col">Rows</th></tr></thead>
+                <tbody id="healthTableCounts"></tbody></table></div></section>
+    </div>
+</div>
+"""

backend/app/frontend/views.py

@@ -5,6 +5,7 @@ from typing import Callable
 
 from app.frontend.miner_detail_view import miner_detail_content
 from app.frontend.monitoring_trends_view import monitoring_trends_content
+from app.frontend.system_health_view import system_health_content
 
 
 NAVIGATION = (
@@ -13,6 +14,7 @@ NAVIGATION = (
     ("analytics", "Trends", "/monitoring/analytics", "miners.view"),
     ("repairs", "Repair Center", "/operations/repairs", "repairs.view"),
     ("miners", "Miners", "/assets/miners", "miners.view"),
+    ("system_health", "System Health", "/system-health", "system_health.view"),
 )
 
 PAGE_DETAILS = {
@@ -46,6 +48,11 @@ PAGE_DETAILS = {
         "Miner Detail",
         "Current identity, operating facts, location, and authorized history.",
     ),
+    "system_health": (
+        "Administration / Diagnostics",
+        "System Health",
+        "Inspect application services, operational data, and maintenance facts.",
+    ),
 }
 
 
@@ -455,7 +462,9 @@ def render_app_page(
         else '<div class="global-search-unavailable" aria-hidden="true"></div>'
     )
     content = (
-        miner_detail_content()
+        system_health_content()
+        if view == "system_health"
+        else miner_detail_content()
         if view == "miner_detail"
         else monitoring_trends_content(has_permission("locations.view"))
         if view == "analytics"
@@ -474,7 +483,9 @@ def render_app_page(
         )
     )
     page_script = (
-        '<script src="/assets/minerops-miner-detail.js" defer></script>'
+        '<script src="/assets/minerops-system-health.js" defer></script>'
+        if view == "system_health"
+        else '<script src="/assets/minerops-miner-detail.js" defer></script>'
         if view == "miner_detail"
         else '<script src="/assets/minerops-monitoring-trends.js" defer></script>'
         if view == "analytics"
@@ -489,7 +500,9 @@ def render_app_page(
         else ""
     )
     page_style = (
-        '<link rel="stylesheet" href="/assets/minerops-monitoring-trends.css">'
+        '<link rel="stylesheet" href="/assets/minerops-system-health.css">'
+        if view == "system_health"
+        else '<link rel="stylesheet" href="/assets/minerops-monitoring-trends.css">'
         if view == "analytics" else ""
     )
     object_attributes = ""

backend/app/main.py

@@ -32,6 +32,8 @@ from app.components.router import router as components_router
 from app.miner_detail.router import router as miner_detail_router
 from app.monitoring.router import router as monitoring_router
 from app.monitoring.frontend import router as monitoring_frontend_router
+from app.system_health.frontend import router as system_health_frontend_router
+from app.system_health.router import router as system_health_router
 from app.repairs.exceptions import (
     ControlBoardReplacementActorRequired,
     InvalidMacAddress,
@@ -67,6 +69,8 @@ app.include_router(dashboard_router)
 app.include_router(miner_detail_router)
 app.include_router(monitoring_frontend_router)
 app.include_router(monitoring_router)
+app.include_router(system_health_frontend_router)
+app.include_router(system_health_router)
 install_identity_middleware(app)
 
 
@@ -34663,744 +34667,6 @@ async def minerops_inject_repair_history_modal(request, call_next):
 
 
 
-# MINEROPS_SYSTEM_HEALTH_V1
-from fastapi.responses import HTMLResponse as MinerOpsSystemHealthHTMLResponse
-from pathlib import Path as MinerOpsSystemPath
-
-
-@app.get("/api/system/health-summary")
-def minerops_system_health_summary(
-    current_user: dict = Depends(get_current_user),
-):
-    import re
-    from collections import Counter
-
-    ensure_location_tables()
-    ensure_maintenance_tables()
-
-    source_path = MinerOpsSystemPath(__file__)
-    source_text = source_path.read_text(errors="ignore")
-    source_lines = source_text.splitlines()
-
-    route_re = re.compile(r'^\s*@app\.(get|post|patch|delete|put)\("([^"]+)"')
-    routes = []
-
-    for line_no, line in enumerate(source_lines, start=1):
-        match = route_re.search(line)
-
-        if match:
-            routes.append(
-                {
-                    "method": match.group(1).upper(),
-                    "path": match.group(2),
-                    "line": line_no,
-                }
-            )
-
-    route_counter = Counter((r["method"], r["path"]) for r in routes)
-    duplicate_routes = []
-
-    for (method, route_path), count in sorted(route_counter.items(), key=lambda item: (item[0][1], item[0][0])):
-        if count > 1:
-            duplicate_routes.append(
-                {
-                    "method": method,
-                    "path": route_path,
-                    "count": count,
-                    "lines": [
-                        r["line"]
-                        for r in routes
-                        if r["method"] == method and r["path"] == route_path
-                    ],
-                }
-            )
-
-    middleware_lines = [
-        line_no
-        for line_no, line in enumerate(source_lines, start=1)
-        if '@app.middleware("http")' in line
-    ]
-
-    asset_routes = [
-        r
-        for r in routes
-        if str(r["path"]).startswith("/assets/")
-    ]
-
-    patch_markers = []
-
-    for line_no, line in enumerate(source_lines, start=1):
-        stripped = line.strip()
-
-        if stripped.startswith("# MINEROPS_") or stripped.startswith("# Step "):
-            patch_markers.append(
-                {
-                    "line": line_no,
-                    "text": stripped,
-                }
-            )
-
-    conn = None
-
-    try:
-        conn = get_db_connection()
-
-        with conn.cursor(row_factory=psycopg.rows.dict_row) as cur:
-            cur.execute(
-                """
-                SELECT
-                    relname AS table_name,
-                    pg_total_relation_size(relid) AS total_bytes,
-                    pg_size_pretty(pg_total_relation_size(relid)) AS total_size
-                FROM pg_catalog.pg_statio_user_tables
-                ORDER BY pg_total_relation_size(relid) DESC
-                LIMIT 15;
-                """
-            )
-            largest_tables = [dict(row) for row in cur.fetchall()]
-
-            cur.execute(
-                """
-                SELECT
-                    relname AS table_name,
-                    n_live_tup::bigint AS estimated_rows,
-                    n_dead_tup::bigint AS estimated_dead_rows,
-                    last_vacuum,
-                    last_autovacuum,
-                    last_analyze,
-                    last_autoanalyze
-                FROM pg_stat_user_tables
-                ORDER BY n_live_tup DESC
-                LIMIT 20;
-                """
-            )
-            row_estimates = [dict(row) for row in cur.fetchall()]
-
-            table_counts = {}
-
-            for table_name in [
-                "miners",
-                "location_positions",
-                "miner_location_states",
-                "miner_maintenance_events",
-                "miner_hashboards",
-                "miner_network_identities",
-                "minerops_agent_command_queue",
-            ]:
-                try:
-                    cur.execute(f"SELECT COUNT(*) AS row_count FROM {table_name};")
-                    table_counts[table_name] = cur.fetchone()["row_count"]
-                except Exception as exc:
-                    table_counts[table_name] = f"error: {exc}"
-
-            cur.execute(
-                """
-                SELECT
-                    COALESCE(status, 'unknown') AS status,
-                    COUNT(*) AS count
-                FROM miners
-                GROUP BY COALESCE(status, 'unknown')
-                ORDER BY status;
-                """
-            )
-            miner_status_counts = [dict(row) for row in cur.fetchall()]
-
-            cur.execute(
-                """
-                SELECT
-                    COALESCE(asset_state, 'unknown') AS asset_state,
-                    COUNT(*) AS count
-                FROM miner_location_states
-                GROUP BY COALESCE(asset_state, 'unknown')
-                ORDER BY asset_state;
-                """
-            )
-            location_state_counts = [dict(row) for row in cur.fetchall()]
-
-            cur.execute(
-                """
-                SELECT
-                    COALESCE(event_status, 'unknown') AS event_status,
-                    COUNT(*) AS count
-                FROM miner_maintenance_events
-                WHERE event_type = 'repair'
-                GROUP BY COALESCE(event_status, 'unknown')
-                ORDER BY event_status;
-                """
-            )
-            repair_status_counts = [dict(row) for row in cur.fetchall()]
-
-            cur.execute(
-                """
-                SELECT
-                    COUNT(*) FILTER (WHERE api3_supported IS TRUE) AS api3_supported,
-                    COUNT(*) FILTER (WHERE api3_supported IS FALSE) AS api3_failed,
-                    COUNT(*) FILTER (WHERE api3_supported IS NULL) AS api3_unknown,
-                    COUNT(*) FILTER (WHERE power_sn IS NOT NULL AND power_sn <> '') AS miners_with_power_sn,
-                    COUNT(*) FILTER (WHERE miner_sn IS NOT NULL AND miner_sn <> '') AS miners_with_miner_sn
-                FROM miners;
-                """
-            )
-            api3_summary = dict(cur.fetchone())
-
-            cur.execute(
-                """
-                SELECT
-                    COUNT(*) AS repair_queue_count
-                FROM miner_location_states
-                WHERE asset_state = 'repair';
-                """
-            )
-            repair_queue_count = cur.fetchone()["repair_queue_count"]
-
-            cur.execute(
-                """
-                SELECT
-                    COUNT(*) AS raw_status_rows,
-                    MIN(observed_at) AS raw_oldest,
-                    MAX(observed_at) AS raw_newest,
-                    COUNT(*) FILTER (WHERE observed_at < now() - interval '30 days') AS raw_older_than_30d
-                FROM miner_status_snapshots;
-                """
-            )
-            raw_status_retention = dict(cur.fetchone())
-
-            cur.execute(
-                """
-                SELECT
-                    COUNT(*) AS hourly_rows,
-                    MIN(bucket_at) AS hourly_oldest,
-                    MAX(bucket_at) AS hourly_newest
-                FROM miner_status_hourly_rollups;
-                """
-            )
-            hourly_retention = dict(cur.fetchone())
-
-            cur.execute(
-                """
-                SELECT
-                    COUNT(*) AS daily_rows,
-                    MIN(bucket_date) AS daily_oldest,
-                    MAX(bucket_date) AS daily_newest
-                FROM miner_status_daily_rollups;
-                """
-            )
-            daily_retention = dict(cur.fetchone())
-
-            cur.execute(
-                """
-                SELECT
-                    mode,
-                    raw_status_retention_days,
-                    raw_status_rows_eligible_delete,
-                    raw_status_rows_deleted,
-                    hourly_rollup_rows_after,
-                    daily_rollup_rows_after,
-                    notes,
-                    started_at,
-                    finished_at
-                FROM minerops_retention_runs
-                ORDER BY started_at DESC
-                LIMIT 1;
-                """
-            )
-            latest_retention_run = cur.fetchone()
-            latest_retention_run = dict(latest_retention_run) if latest_retention_run else None
-
-        return api_response(
-            data={
-                "app": {
-                    "status": "ok",
-                    "source_lines": len(source_lines),
-                    "source_size_bytes": source_path.stat().st_size,
-                    "route_count": len(routes),
-                    "duplicate_route_count": len(duplicate_routes),
-                    "middleware_count": len(middleware_lines),
-                    "asset_route_count": len(asset_routes),
-                },
-                "routes": {
-                    "duplicates": duplicate_routes,
-                    "middleware_lines": middleware_lines,
-                    "asset_routes": asset_routes,
-                    "recent_patch_markers": patch_markers[-40:],
-                },
-                "database": {
-                    "largest_tables": largest_tables,
-                    "row_estimates": row_estimates,
-                    "table_counts": table_counts,
-                    "miner_status_counts": miner_status_counts,
-                    "location_state_counts": location_state_counts,
-                    "repair_status_counts": repair_status_counts,
-                    "api3_summary": api3_summary,
-                    "repair_queue_count": repair_queue_count,
-                    "retention": {
-                        "raw_status": raw_status_retention,
-                        "hourly_rollups": hourly_retention,
-                        "daily_rollups": daily_retention,
-                        "latest_run": latest_retention_run,
-                    },
-                },
-            }
-        )
-
-    finally:
-        if conn is not None:
-            conn.close()
-
-
-MINEROPS_SYSTEM_HEALTH_HTML = r"""
-<!DOCTYPE html>
-<html>
-<head>
-    <title>MinerOps System Health</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <style>
-        body {
-            font-family: Segoe UI, Arial, sans-serif;
-            background: #f4f7fb;
-            color: #102a43;
-            margin: 0;
-            padding: 22px;
-        }
-
-        .nav {
-            display: flex;
-            flex-wrap: wrap;
-            gap: 8px;
-            margin-bottom: 16px;
-        }
-
-        .nav a {
-            border: 1px solid #bcccdc;
-            border-radius: 999px;
-            padding: 7px 11px;
-            background: #ffffff;
-            color: #0b63ce;
-            font-weight: 700;
-            text-decoration: none;
-            font-size: 13px;
-        }
-
-        .nav a.active {
-            background: #0f766e;
-            border-color: #0f766e;
-            color: #ffffff;
-        }
-
-        h1 {
-            margin: 0 0 4px 0;
-            font-size: 28px;
-        }
-
-        h2 {
-            margin-top: 0;
-        }
-
-        .muted {
-            color: #52606d;
-            font-size: 13px;
-        }
-
-        .card {
-            background: #ffffff;
-            border: 1px solid #d9e2ec;
-            border-radius: 12px;
-            padding: 14px;
-            margin-bottom: 14px;
-            box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08);
-        }
-
-        .metrics {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(145px, 1fr));
-            gap: 10px;
-        }
-
-        .metric {
-            border: 1px solid #d9e2ec;
-            border-radius: 10px;
-            background: #f8fafc;
-            padding: 11px;
-        }
-
-        .metric .value {
-            font-size: 24px;
-            font-weight: 800;
-            margin-top: 2px;
-            overflow-wrap: anywhere;
-        }
-
-        table {
-            width: 100%;
-            border-collapse: collapse;
-            font-size: 13px;
-        }
-
-        th, td {
-            border-bottom: 1px solid #d9e2ec;
-            padding: 7px;
-            text-align: left;
-            vertical-align: top;
-        }
-
-        th {
-            background: #f1f5f9;
-        }
-
-        .table-wrap {
-            overflow-x: auto;
-        }
-
-        button {
-            border: 1px solid #829ab1;
-            border-radius: 8px;
-            background: #f8fafc;
-            padding: 8px 10px;
-            font-weight: 700;
-            cursor: pointer;
-        }
-
-        button.primary {
-            background: #0f766e;
-            border-color: #0f766e;
-            color: #ffffff;
-        }
-
-        .status {
-            margin: 12px 0;
-            padding: 10px;
-            border-radius: 8px;
-            font-size: 13px;
-        }
-
-        .ok {
-            background: #f0fdf4;
-            border: 1px solid #bbf7d0;
-            color: #166534;
-        }
-
-        .bad {
-            background: #fef2f2;
-            border: 1px solid #fecaca;
-            color: #991b1b;
-        }
-
-        .grid-two {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
-            gap: 14px;
-        }
-
-        code {
-            background: #e2e8f0;
-            padding: 1px 4px;
-            border-radius: 4px;
-        }
-    </style>
-</head>
-<body>
-    <div class="nav">
-        <a href="/dashboard">Dashboard</a>
-        <a href="/repairs">Repairs</a>
-        <a href="/locations">Locations</a>
-        <a class="active" href="/system-health">System Health</a>
-    </div>
-
-    <div style="display:flex;justify-content:space-between;gap:12px;align-items:start;flex-wrap:wrap;margin-bottom:14px;">
-        <div>
-            <h1>System Health</h1>
-            <div class="muted">Quick operational view of MinerOps app size, routes, database tables, repairs, and location state.</div>
-        </div>
-        <button id="refreshBtn" type="button" class="primary">Refresh</button>
-    </div>
-
-    <div id="statusBox" class="status ok">Loading system health...</div>
-
-    <div class="card">
-        <h2>App Health</h2>
-        <div class="metrics">
-            <div class="metric"><div class="muted">Status</div><div id="appStatus" class="value">-</div></div>
-            <div class="metric"><div class="muted">main.py lines</div><div id="sourceLines" class="value">-</div></div>
-            <div class="metric"><div class="muted">Routes</div><div id="routeCount" class="value">-</div></div>
-            <div class="metric"><div class="muted">Duplicate routes</div><div id="duplicateRoutes" class="value">-</div></div>
-            <div class="metric"><div class="muted">Middleware</div><div id="middlewareCount" class="value">-</div></div>
-            <div class="metric"><div class="muted">Assets</div><div id="assetRouteCount" class="value">-</div></div>
-        </div>
-    </div>
-
-    <div class="card">
-        <h2>Fleet / Repair State</h2>
-        <div class="metrics">
-            <div class="metric"><div class="muted">Miners</div><div id="minerCount" class="value">-</div></div>
-            <div class="metric"><div class="muted">In Repair</div><div id="repairCount" class="value">-</div></div>
-            <div class="metric"><div class="muted">Shelved</div><div id="shelvedCount" class="value">-</div></div>
-            <div class="metric"><div class="muted">Waiting Placement</div><div id="waitingCount" class="value">-</div></div>
-            <div class="metric"><div class="muted">API3 Supported</div><div id="api3Supported" class="value">-</div></div>
-            <div class="metric"><div class="muted">API3 Failed</div><div id="api3Failed" class="value">-</div></div>
-        </div>
-    </div>
-
-    <div class="card">
-        <h2>Status Retention</h2>
-        <div class="metrics">
-            <div class="metric"><div class="muted">Raw Status Rows</div><div id="rawStatusRows" class="value">-</div></div>
-            <div class="metric"><div class="muted">Older Than 30 Days</div><div id="rawOlder30" class="value">-</div></div>
-            <div class="metric"><div class="muted">Hourly Rollups</div><div id="hourlyRollups" class="value">-</div></div>
-            <div class="metric"><div class="muted">Daily Rollups</div><div id="dailyRollups" class="value">-</div></div>
-            <div class="metric"><div class="muted">Last Retention Mode</div><div id="lastRetentionMode" class="value">-</div></div>
-            <div class="metric"><div class="muted">Last Deleted Rows</div><div id="lastDeletedRows" class="value">-</div></div>
-        </div>
-        <div id="retentionDetails" class="muted" style="margin-top:10px;">Loading retention details...</div>
-    </div>
-
-    <div class="grid-two">
-        <div class="card">
-            <h2>Largest Tables</h2>
-            <div class="table-wrap">
-                <table>
-                    <thead><tr><th>Table</th><th>Size</th></tr></thead>
-                    <tbody id="largestTablesBody"><tr><td colspan="2">Loading...</td></tr></tbody>
-                </table>
-            </div>
-        </div>
-
-        <div class="card">
-            <h2>Repair Status Counts</h2>
-            <div class="table-wrap">
-                <table>
-                    <thead><tr><th>Status</th><th>Count</th></tr></thead>
-                    <tbody id="repairStatusBody"><tr><td colspan="2">Loading...</td></tr></tbody>
-                </table>
-            </div>
-        </div>
-    </div>
-
-    <div class="grid-two">
-        <div class="card">
-            <h2>Duplicate Routes</h2>
-            <div class="table-wrap">
-                <table>
-                    <thead><tr><th>Method</th><th>Path</th><th>Count</th><th>Lines</th></tr></thead>
-                    <tbody id="duplicateRoutesBody"><tr><td colspan="4">Loading...</td></tr></tbody>
-                </table>
-            </div>
-        </div>
-
-        <div class="card">
-            <h2>Recent Patch Markers</h2>
-            <div class="table-wrap">
-                <table>
-                    <thead><tr><th>Line</th><th>Marker</th></tr></thead>
-                    <tbody id="patchMarkersBody"><tr><td colspan="2">Loading...</td></tr></tbody>
-                </table>
-            </div>
-        </div>
-    </div>
-
-<script>
-(function () {
-    "use strict";
-
-    function getStoredToken() {
-        const keys = [
-            "minerops_token",
-            "mineropsToken",
-            "authToken",
-            "access_token",
-            "token"
-        ];
-
-        for (const storage of [window.localStorage, window.sessionStorage]) {
-            if (!storage) {
-                continue;
-            }
-
-            for (const key of keys) {
-                const value = storage.getItem(key);
-
-                if (value) {
-                    return value;
-                }
-            }
-        }
-
-        return "";
-    }
-
-    function authHeaders(extra) {
-        const headers = Object.assign({}, extra || {});
-        const token = getStoredToken();
-
-        if (token && !headers.Authorization) {
-            headers.Authorization = "Bearer " + token;
-        }
-
-        return headers;
-    }
-
-    async function apiGet(url) {
-        const response = await fetch(url, {
-            method: "GET",
-            credentials: "same-origin",
-            headers: authHeaders({
-                "Accept": "application/json"
-            })
-        });
-
-        const payload = await response.json().catch(function () {
-            return null;
-        });
-
-        if (!response.ok || (payload && payload.error)) {
-            throw new Error((payload && payload.error) || "Request failed");
-        }
-
-        return payload ? payload.data : null;
-    }
-
-    function escapeHtml(value) {
-        return String(value == null ? "" : value)
-            .replace(/&/g, "&amp;")
-            .replace(/</g, "&lt;")
-            .replace(/>/g, "&gt;")
-            .replace(/\"/g, "&quot;")
-            .replace(/'/g, "&#039;");
-    }
-
-    function setText(id, value) {
-        const el = document.getElementById(id);
-
-        if (el) {
-            el.textContent = value == null ? "-" : value;
-        }
-    }
-
-    function tableRows(rows, columns, emptyMessage) {
-        if (!rows || !rows.length) {
-            return '<tr><td colspan="' + columns.length + '">' + escapeHtml(emptyMessage || "No rows") + '</td></tr>';
-        }
-
-        return rows.map(function (row) {
-            return "<tr>" + columns.map(function (col) {
-                let value = row[col];
-
-                if (Array.isArray(value)) {
-                    value = value.join(", ");
-                }
-
-                return "<td>" + escapeHtml(value) + "</td>";
-            }).join("") + "</tr>";
-        }).join("");
-    }
-
-    function countByKey(rows, keyName, valueName) {
-        const result = {};
-
-        for (const row of rows || []) {
-            result[row[keyName]] = row[valueName];
-        }
-
-        return result;
-    }
-
-    async function loadHealth() {
-        const statusBox = document.getElementById("statusBox");
-
-        try {
-            statusBox.className = "status ok";
-            statusBox.textContent = "Loading system health...";
-
-            const data = await apiGet("/api/system/health-summary");
-            const app = data.app || {};
-            const db = data.database || {};
-            const routes = data.routes || {};
-
-            setText("appStatus", app.status || "-");
-            setText("sourceLines", Number(app.source_lines || 0).toLocaleString());
-            setText("routeCount", app.route_count || 0);
-            setText("duplicateRoutes", app.duplicate_route_count || 0);
-            setText("middlewareCount", app.middleware_count || 0);
-            setText("assetRouteCount", app.asset_route_count || 0);
-
-            const locationCounts = countByKey(db.location_state_counts, "asset_state", "count");
-            const tableCounts = db.table_counts || {};
-            const api3 = db.api3_summary || {};
-
-            const retention = db.retention || {};
-            const rawStatus = retention.raw_status || {};
-            const hourly = retention.hourly_rollups || {};
-            const daily = retention.daily_rollups || {};
-            const latestRetention = retention.latest_run || {};
-
-            setText("minerCount", tableCounts.miners || 0);
-            setText("repairCount", db.repair_queue_count || 0);
-            setText("shelvedCount", locationCounts.shelved || 0);
-            setText("waitingCount", locationCounts.waiting_placement || 0);
-            setText("api3Supported", api3.api3_supported || 0);
-            setText("api3Failed", api3.api3_failed || 0);
-
-            setText("rawStatusRows", Number(rawStatus.raw_status_rows || 0).toLocaleString());
-            setText("rawOlder30", Number(rawStatus.raw_older_than_30d || 0).toLocaleString());
-            setText("hourlyRollups", Number(hourly.hourly_rows || 0).toLocaleString());
-            setText("dailyRollups", Number(daily.daily_rows || 0).toLocaleString());
-            setText("lastRetentionMode", latestRetention.mode || "-");
-            setText("lastDeletedRows", Number(latestRetention.raw_status_rows_deleted || 0).toLocaleString());
-
-            const retentionDetails = document.getElementById("retentionDetails");
-            if (retentionDetails) {
-                retentionDetails.innerHTML =
-                    "Raw range: <code>" + escapeHtml(rawStatus.raw_oldest || "-") + "</code> to <code>" + escapeHtml(rawStatus.raw_newest || "-") + "</code><br>" +
-                    "Hourly range: <code>" + escapeHtml(hourly.hourly_oldest || "-") + "</code> to <code>" + escapeHtml(hourly.hourly_newest || "-") + "</code><br>" +
-                    "Daily range: <code>" + escapeHtml(daily.daily_oldest || "-") + "</code> to <code>" + escapeHtml(daily.daily_newest || "-") + "</code><br>" +
-                    "Latest run: <code>" + escapeHtml(latestRetention.started_at || "-") + "</code> | " + escapeHtml(latestRetention.notes || "-");
-            }
-
-            document.getElementById("largestTablesBody").innerHTML = tableRows(
-                db.largest_tables || [],
-                ["table_name", "total_size"],
-                "No table size data"
-            );
-
-            document.getElementById("repairStatusBody").innerHTML = tableRows(
-                db.repair_status_counts || [],
-                ["event_status", "count"],
-                "No repair events"
-            );
-
-            document.getElementById("duplicateRoutesBody").innerHTML = tableRows(
-                routes.duplicates || [],
-                ["method", "path", "count", "lines"],
-                "No duplicate routes"
-            );
-
-            document.getElementById("patchMarkersBody").innerHTML = tableRows(
-                routes.recent_patch_markers || [],
-                ["line", "text"],
-                "No patch markers"
-            );
-
-            statusBox.className = "status ok";
-            statusBox.textContent = "System health loaded.";
-        } catch (error) {
-            statusBox.className = "status bad";
-            statusBox.textContent = "System health failed to load: " + (error.message || error);
-        }
-    }
-
-    document.getElementById("refreshBtn").addEventListener("click", loadHealth);
-    loadHealth();
-})();
-</script>
-</body>
-</html>
-"""
-
-
-@app.get("/system-health", response_class=MinerOpsSystemHealthHTMLResponse)
-def minerops_system_health_page():
-    return MINEROPS_SYSTEM_HEALTH_HTML
-
-
 # MINEROPS_SYSTEM_HEALTH_RETENTION_V1
 
 # MINEROPS_REPAIR_READY_TO_SHELF_STATUS_V1

backend/app/system_health/__init__.py

@@ -0,0 +1 @@
+"""System Health diagnostics domain."""

backend/app/system_health/adapters.py

@@ -0,0 +1,29 @@
+"""External service adapters for System Health."""
+
+from collections.abc import Callable
+
+import redis
+
+from app.config import settings
+
+
+RedisFactory = Callable[[], redis.Redis]
+
+
+def default_redis_factory() -> redis.Redis:
+    return redis.Redis.from_url(
+        settings.redis_url,
+        socket_connect_timeout=2,
+        socket_timeout=2,
+    )
+
+
+class RedisHealthAdapter:
+    def __init__(self, factory: RedisFactory | None = None):
+        self._factory = factory or default_redis_factory
+
+    def ping(self) -> bool:
+        try:
+            return bool(self._factory().ping())
+        except Exception:
+            return False

backend/app/system_health/application.py

@@ -0,0 +1,42 @@
+"""Startup-cached, non-secret application metadata."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+
+from app.config import settings
+
+
+def _clean_build_value(value: str | None) -> str | None:
+    if value is None:
+        return None
+    cleaned = value.strip()
+    return cleaned[:128] if cleaned else None
+
+
+def _expected_migration_head() -> str | None:
+    try:
+        backend_root = Path(__file__).resolve().parents[2]
+        config = Config(str(backend_root / "alembic.ini"))
+        config.set_main_option("script_location", str(backend_root / "alembic"))
+        return ScriptDirectory.from_config(config).get_current_head()
+    except Exception:
+        return None
+
+
+@dataclass(frozen=True)
+class ApplicationMetadata:
+    environment: str
+    build_commit: str | None
+    build_version: str | None
+    expected_migration_head: str | None
+
+
+APPLICATION_METADATA = ApplicationMetadata(
+    environment=settings.app_env,
+    build_commit=_clean_build_value(settings.minerops_build_commit),
+    build_version=_clean_build_value(settings.minerops_build_version),
+    expected_migration_head=_expected_migration_head(),
+)

backend/app/system_health/dependencies.py

@@ -0,0 +1,9 @@
+"""Dependency construction for System Health."""
+
+from app.system_health.adapters import RedisHealthAdapter
+from app.system_health.repository import SystemHealthRepository
+from app.system_health.service import SystemHealthService
+
+
+def get_system_health_service() -> SystemHealthService:
+    return SystemHealthService(SystemHealthRepository(), RedisHealthAdapter())

backend/app/system_health/frontend.py

@@ -0,0 +1,35 @@
+"""Authenticated frontend routes for System Health."""
+
+from pathlib import Path
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import HTMLResponse, Response
+
+from app.frontend.views import render_app_page
+from app.identity import require_permission, user_has_permission
+
+
+router = APIRouter(tags=["frontend"])
+STATIC_ROOT = Path(__file__).parents[1] / "frontend" / "static"
+SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))]
+
+
+@router.get("/assets/minerops-system-health.css", include_in_schema=False)
+def system_health_styles() -> Response:
+    return Response((STATIC_ROOT / "system-health.css").read_text(encoding="utf-8"),
+                    media_type="text/css", headers={"Cache-Control": "no-store"})
+
+
+@router.get("/assets/minerops-system-health.js", include_in_schema=False)
+def system_health_script() -> Response:
+    return Response((STATIC_ROOT / "system-health.js").read_text(encoding="utf-8"),
+                    media_type="text/javascript", headers={"Cache-Control": "no-store"})
+
+
+@router.get("/system-health", response_class=HTMLResponse, include_in_schema=False)
+def system_health_page(current_user: SystemHealthUser) -> str:
+    return render_app_page(
+        "system_health", current_user,
+        lambda permission: user_has_permission(current_user, permission),
+    )

backend/app/system_health/models.py

@@ -0,0 +1,124 @@
+"""Typed contracts for the System Health overview."""
+
+from datetime import date, datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+Availability = Literal["available", "unavailable"]
+MigrationState = Literal["current", "mismatch", "unavailable"]
+
+
+class ApplicationDiagnostics(BaseModel):
+    status: Literal["available"] = "available"
+    environment: str
+    build_commit: str | None = None
+    build_version: str | None = None
+    server_time: datetime
+    expected_migration_head: str | None = None
+
+
+class MigrationDiagnostics(BaseModel):
+    state: MigrationState
+    current_revision: str | None = None
+    expected_revision: str | None = None
+
+
+class TableCount(BaseModel):
+    name: str
+    row_count: int
+
+
+class TableSize(BaseModel):
+    name: str
+    total_bytes: int
+
+
+class TelemetryDiagnostics(BaseModel):
+    snapshot_count: int
+    oldest_observed_at: datetime | None = None
+    latest_observed_at: datetime | None = None
+
+
+class FleetIntegrityDiagnostics(BaseModel):
+    total_miners: int
+    miners_without_asset_id: int
+    unmapped_miners: int
+    miners_in_multiple_positions: int
+    duplicate_current_mac_groups: int
+
+
+class LocationIntegrityDiagnostics(BaseModel):
+    sites: int
+    rooms: int
+    positions: int
+    occupied_positions: int
+    empty_positions: int
+    unscoped_legacy_positions: int
+    expected_mac_mismatches: int
+
+
+class RepairIntegrityDiagnostics(BaseModel):
+    active_modular_repairs: int
+    duplicate_active_repair_groups: int
+    legacy_repair_only_miners: int
+    lifecycle_repair_without_active_repair: int
+    active_repair_lifecycle_mismatches: int
+    waiting_placement_records: int
+    repair_events: int
+    repair_parts: int
+
+
+class RollupDiagnostics(BaseModel):
+    table_present: bool
+    row_count: int | None = None
+    latest_bucket: datetime | date | None = None
+    population: Literal["manual"] = "manual"
+
+
+class RetentionDiagnostics(BaseModel):
+    raw_retention_enabled: Literal[False] = False
+    raw_retention_mode: Literal["not_enabled"] = "not_enabled"
+    hourly: RollupDiagnostics
+    daily: RollupDiagnostics
+    latest_recorded_run_at: datetime | None = None
+    latest_recorded_run_mode: str | None = None
+    completeness: Literal["not_guaranteed"] = "not_guaranteed"
+
+
+class PostgreSQLDiagnostics(BaseModel):
+    status: Availability
+    migration: MigrationDiagnostics
+    table_counts: list[TableCount] = Field(default_factory=list)
+    largest_tables: list[TableSize] = Field(default_factory=list)
+    telemetry: TelemetryDiagnostics | None = None
+    fleet: FleetIntegrityDiagnostics | None = None
+    locations: LocationIntegrityDiagnostics | None = None
+    repairs: RepairIntegrityDiagnostics | None = None
+    retention: RetentionDiagnostics | None = None
+
+
+class RedisDiagnostics(BaseModel):
+    status: Availability
+
+
+class SystemHealthOverviewData(BaseModel):
+    application: ApplicationDiagnostics
+    postgresql: PostgreSQLDiagnostics
+    redis: RedisDiagnostics
+
+
+class SystemHealthOverviewMeta(BaseModel):
+    generated_at: datetime
+    partial: bool
+    unavailable_sections: list[str] = Field(default_factory=list)
+    frontend_request_count: Literal[1] = 1
+    postgresql_statement_count: Literal[3] = 3
+    redis_call_count: Literal[1] = 1
+
+
+class SystemHealthOverviewResponse(BaseModel):
+    data: SystemHealthOverviewData
+    meta: SystemHealthOverviewMeta
+    error: None = None

backend/app/system_health/repository.py

@@ -0,0 +1,179 @@
+"""PostgreSQL diagnostics for System Health."""
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from typing import Any
+
+import psycopg
+from psycopg.rows import dict_row
+
+from app.config import settings
+
+
+ConnectionFactory = Callable[[], psycopg.Connection]
+
+
+def default_connection_factory() -> psycopg.Connection:
+    return psycopg.connect(
+        host=settings.postgres_host,
+        port=settings.postgres_port,
+        dbname=settings.postgres_db,
+        user=settings.postgres_user,
+        password=settings.postgres_password,
+        connect_timeout=3,
+    )
+
+
+MIGRATION_SQL = """
+SELECT clock_timestamp() AS database_time,
+       (SELECT version_num FROM alembic_version LIMIT 1) AS current_revision
+"""
+
+OPERATIONS_SQL = """
+WITH telemetry AS (
+    SELECT count(*)::bigint AS snapshot_count,
+           min(observed_at) AS oldest_observed_at,
+           max(observed_at) AS latest_observed_at
+    FROM miner_status_snapshots
+),
+hourly AS (
+    SELECT count(*)::bigint AS row_count, max(bucket_at) AS latest_bucket
+    FROM miner_status_hourly_rollups
+),
+daily AS (
+    SELECT count(*)::bigint AS row_count, max(bucket_date) AS latest_bucket
+    FROM miner_status_daily_rollups
+),
+latest_retention AS (
+    SELECT mode, started_at FROM minerops_retention_runs
+    ORDER BY started_at DESC LIMIT 1
+),
+largest AS (
+    SELECT coalesce(jsonb_agg(to_jsonb(sized) ORDER BY sized.total_bytes DESC), '[]'::jsonb) AS rows
+    FROM (
+        SELECT relname AS name, pg_total_relation_size(relid)::bigint AS total_bytes
+        FROM pg_catalog.pg_statio_user_tables
+        WHERE relname = ANY(%s)
+        ORDER BY pg_total_relation_size(relid) DESC, relname
+        LIMIT 8
+    ) AS sized
+)
+SELECT
+    (SELECT count(*) FROM miners)::bigint AS miners,
+    (SELECT count(*) FROM sites)::bigint AS sites,
+    (SELECT count(*) FROM rooms)::bigint AS rooms,
+    (SELECT count(*) FROM location_positions)::bigint AS location_positions,
+    (SELECT count(*) FROM repairs)::bigint AS repairs,
+    (SELECT count(*) FROM repair_events)::bigint AS repair_events,
+    (SELECT count(*) FROM repair_parts)::bigint AS repair_parts,
+    telemetry.snapshot_count, telemetry.oldest_observed_at,
+    telemetry.latest_observed_at,
+    to_regclass('public.miner_status_hourly_rollups') IS NOT NULL AS hourly_present,
+    hourly.row_count AS hourly_rows, hourly.latest_bucket AS hourly_latest,
+    to_regclass('public.miner_status_daily_rollups') IS NOT NULL AS daily_present,
+    daily.row_count AS daily_rows, daily.latest_bucket AS daily_latest,
+    latest_retention.mode AS retention_mode,
+    latest_retention.started_at AS retention_started_at,
+    largest.rows AS largest_tables
+FROM telemetry CROSS JOIN hourly CROSS JOIN daily CROSS JOIN largest
+LEFT JOIN latest_retention ON true
+"""
+
+INTEGRITY_SQL = """
+WITH position_counts AS (
+    SELECT miner_id, count(*) AS position_count
+    FROM location_positions WHERE miner_id IS NOT NULL GROUP BY miner_id
+),
+active_repairs AS (
+    SELECT id, miner_id FROM repairs
+    WHERE status IN ('OPEN', 'IN_PROGRESS', 'WAITING_FOR_PARTS', 'READY_FOR_TEST')
+),
+active_repair_groups AS (
+    SELECT miner_id FROM active_repairs GROUP BY miner_id HAVING count(*) > 1
+),
+normalized_macs AS (
+    SELECT lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g')) AS mac
+    FROM miners WHERE current_mac IS NOT NULL AND trim(current_mac) <> ''
+    GROUP BY lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g'))
+    HAVING count(*) > 1
+)
+SELECT
+    (SELECT count(*) FROM miners WHERE asset_tag IS NULL OR trim(asset_tag) = '')::bigint
+        AS miners_without_asset_id,
+    (SELECT count(*) FROM miners m WHERE NOT EXISTS (
+        SELECT 1 FROM location_positions p WHERE p.miner_id = m.id
+    ))::bigint AS unmapped_miners,
+    (SELECT count(*) FROM position_counts WHERE position_count > 1)::bigint
+        AS miners_in_multiple_positions,
+    (SELECT count(*) FROM normalized_macs)::bigint AS duplicate_current_mac_groups,
+    (SELECT count(*) FROM location_positions WHERE room_id IS NULL)::bigint
+        AS unscoped_legacy_positions,
+    (SELECT count(*) FROM location_positions p JOIN miners m ON m.id = p.miner_id
+     WHERE p.expected_mac IS NOT NULL AND trim(p.expected_mac) <> ''
+       AND m.current_mac IS NOT NULL AND trim(m.current_mac) <> ''
+       AND lower(regexp_replace(trim(p.expected_mac), '[^0-9A-Fa-f]', '', 'g'))
+           <> lower(regexp_replace(trim(m.current_mac), '[^0-9A-Fa-f]', '', 'g'))
+    )::bigint AS expected_mac_mismatches,
+    (SELECT count(*) FROM location_positions WHERE miner_id IS NOT NULL)::bigint
+        AS occupied_positions,
+    (SELECT count(*) FROM location_positions WHERE miner_id IS NULL)::bigint
+        AS empty_positions,
+    (SELECT count(*) FROM active_repairs)::bigint AS active_modular_repairs,
+    (SELECT count(*) FROM active_repair_groups)::bigint AS duplicate_active_repair_groups,
+    (SELECT count(*) FROM miner_location_states state
+     WHERE state.asset_state = 'repair' AND NOT EXISTS (
+         SELECT 1 FROM active_repairs repair WHERE repair.miner_id = state.miner_id
+     ))::bigint AS legacy_repair_only_miners,
+    (SELECT count(*) FROM miners m WHERE m.lifecycle_status = 'in_repair'
+     AND NOT EXISTS (SELECT 1 FROM active_repairs r WHERE r.miner_id = m.id))::bigint
+        AS lifecycle_repair_without_active_repair,
+    (SELECT count(*) FROM active_repairs r JOIN miners m ON m.id = r.miner_id
+     WHERE m.lifecycle_status <> 'in_repair')::bigint
+        AS active_repair_lifecycle_mismatches,
+    (SELECT count(*) FROM miner_location_states WHERE asset_state = 'waiting_placement')::bigint
+        AS waiting_placement_records
+"""
+
+
+RELEVANT_TABLES = [
+    "miner_status_snapshots", "repair_events", "repair_parts", "repairs",
+    "location_positions", "miners", "miner_status_hourly_rollups",
+    "miner_status_daily_rollups",
+]
+
+
+class SystemHealthRepository:
+    def __init__(
+        self,
+        connection_factory: ConnectionFactory | None = None,
+        *,
+        connection: psycopg.Connection | None = None,
+    ):
+        self._connection_factory = connection_factory or default_connection_factory
+        self._connection = connection
+
+    @contextmanager
+    def _connection_scope(self) -> Iterator[psycopg.Connection]:
+        if self._connection is not None:
+            yield self._connection
+            return
+        connection = self._connection_factory()
+        try:
+            yield connection
+        finally:
+            connection.close()
+
+    def read_overview(self) -> dict[str, Any]:
+        with self._connection_scope() as connection:
+            with connection.cursor(row_factory=dict_row) as cur:
+                cur.execute(MIGRATION_SQL)
+                migration = dict(cur.fetchone())
+                cur.execute(OPERATIONS_SQL, (RELEVANT_TABLES,))
+                operations = dict(cur.fetchone())
+                cur.execute(INTEGRITY_SQL)
+                integrity = dict(cur.fetchone())
+        return {
+            "migration": migration,
+            "operations": operations,
+            "integrity": integrity,
+        }

backend/app/system_health/router.py

@@ -0,0 +1,27 @@
+"""HTTP boundary for the System Health overview."""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Response
+
+from app.identity import require_permission
+from app.system_health.dependencies import get_system_health_service
+from app.system_health.models import SystemHealthOverviewResponse
+from app.system_health.service import SystemHealthService
+
+
+router = APIRouter(prefix="/api/system", tags=["system-health"])
+SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))]
+SystemHealthServiceDependency = Annotated[
+    SystemHealthService, Depends(get_system_health_service)
+]
+
+
+@router.get("/health-summary", response_model=SystemHealthOverviewResponse)
+def get_system_health_overview(
+    current_user: SystemHealthUser,
+    service: SystemHealthServiceDependency,
+    response: Response,
+) -> SystemHealthOverviewResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.get_overview()

backend/app/system_health/service.py

@@ -0,0 +1,136 @@
+"""Composition and failure semantics for System Health diagnostics."""
+
+from datetime import datetime, timezone
+
+from app.system_health.adapters import RedisHealthAdapter
+from app.system_health.application import APPLICATION_METADATA, ApplicationMetadata
+from app.system_health.models import (
+    ApplicationDiagnostics, FleetIntegrityDiagnostics, LocationIntegrityDiagnostics,
+    MigrationDiagnostics, PostgreSQLDiagnostics, RedisDiagnostics,
+    RepairIntegrityDiagnostics, RetentionDiagnostics, RollupDiagnostics,
+    SystemHealthOverviewData, SystemHealthOverviewMeta,
+    SystemHealthOverviewResponse, TableCount, TableSize, TelemetryDiagnostics,
+)
+from app.system_health.repository import SystemHealthRepository
+
+
+class SystemHealthService:
+    def __init__(self, repository: SystemHealthRepository,
+                 redis_adapter: RedisHealthAdapter,
+                 metadata: ApplicationMetadata = APPLICATION_METADATA):
+        self.repository = repository
+        self.redis_adapter = redis_adapter
+        self.metadata = metadata
+
+    def get_overview(self) -> SystemHealthOverviewResponse:
+        generated_at = datetime.now(timezone.utc)
+        unavailable = []
+        try:
+            postgresql = self._postgresql_diagnostics(
+                self.repository.read_overview()
+            )
+        except Exception:
+            unavailable.append("postgresql")
+            postgresql = PostgreSQLDiagnostics(
+                status="unavailable",
+                migration=MigrationDiagnostics(
+                    state="unavailable",
+                    expected_revision=self.metadata.expected_migration_head,
+                ),
+            )
+        redis_available = self.redis_adapter.ping()
+        if not redis_available:
+            unavailable.append("redis")
+        return SystemHealthOverviewResponse(
+            data=SystemHealthOverviewData(
+                application=ApplicationDiagnostics(
+                    environment=self.metadata.environment,
+                    build_commit=self.metadata.build_commit,
+                    build_version=self.metadata.build_version,
+                    server_time=generated_at,
+                    expected_migration_head=self.metadata.expected_migration_head,
+                ),
+                postgresql=postgresql,
+                redis=RedisDiagnostics(
+                    status="available" if redis_available else "unavailable"
+                ),
+            ),
+            meta=SystemHealthOverviewMeta(
+                generated_at=generated_at,
+                partial=bool(unavailable),
+                unavailable_sections=unavailable,
+            ),
+        )
+
+    def _postgresql_diagnostics(self, payload: dict) -> PostgreSQLDiagnostics:
+        migration = payload["migration"]
+        operations = payload["operations"]
+        integrity = payload["integrity"]
+        current = migration.get("current_revision")
+        expected = self.metadata.expected_migration_head
+        state = (
+            "unavailable" if not current or not expected
+            else "current" if current == expected else "mismatch"
+        )
+        counts = [
+            TableCount(name=name, row_count=int(operations[name]))
+            for name in (
+                "miners", "sites", "rooms", "location_positions", "repairs",
+                "repair_events", "repair_parts", "snapshot_count",
+            )
+        ]
+        return PostgreSQLDiagnostics(
+            status="available",
+            migration=MigrationDiagnostics(
+                state=state, current_revision=current, expected_revision=expected
+            ),
+            table_counts=counts,
+            largest_tables=[
+                TableSize(name=row["name"], total_bytes=int(row["total_bytes"]))
+                for row in operations.get("largest_tables") or []
+            ],
+            telemetry=TelemetryDiagnostics(
+                snapshot_count=int(operations["snapshot_count"]),
+                oldest_observed_at=operations.get("oldest_observed_at"),
+                latest_observed_at=operations.get("latest_observed_at"),
+            ),
+            fleet=FleetIntegrityDiagnostics(
+                total_miners=int(operations["miners"]),
+                miners_without_asset_id=int(integrity["miners_without_asset_id"]),
+                unmapped_miners=int(integrity["unmapped_miners"]),
+                miners_in_multiple_positions=int(integrity["miners_in_multiple_positions"]),
+                duplicate_current_mac_groups=int(integrity["duplicate_current_mac_groups"]),
+            ),
+            locations=LocationIntegrityDiagnostics(
+                sites=int(operations["sites"]), rooms=int(operations["rooms"]),
+                positions=int(operations["location_positions"]),
+                occupied_positions=int(integrity["occupied_positions"]),
+                empty_positions=int(integrity["empty_positions"]),
+                unscoped_legacy_positions=int(integrity["unscoped_legacy_positions"]),
+                expected_mac_mismatches=int(integrity["expected_mac_mismatches"]),
+            ),
+            repairs=RepairIntegrityDiagnostics(
+                active_modular_repairs=int(integrity["active_modular_repairs"]),
+                duplicate_active_repair_groups=int(integrity["duplicate_active_repair_groups"]),
+                legacy_repair_only_miners=int(integrity["legacy_repair_only_miners"]),
+                lifecycle_repair_without_active_repair=int(integrity["lifecycle_repair_without_active_repair"]),
+                active_repair_lifecycle_mismatches=int(integrity["active_repair_lifecycle_mismatches"]),
+                waiting_placement_records=int(integrity["waiting_placement_records"]),
+                repair_events=int(operations["repair_events"]),
+                repair_parts=int(operations["repair_parts"]),
+            ),
+            retention=RetentionDiagnostics(
+                hourly=RollupDiagnostics(
+                    table_present=bool(operations["hourly_present"]),
+                    row_count=int(operations["hourly_rows"]),
+                    latest_bucket=operations.get("hourly_latest"),
+                ),
+                daily=RollupDiagnostics(
+                    table_present=bool(operations["daily_present"]),
+                    row_count=int(operations["daily_rows"]),
+                    latest_bucket=operations.get("daily_latest"),
+                ),
+                latest_recorded_run_at=operations.get("retention_started_at"),
+                latest_recorded_run_mode=operations.get("retention_mode"),
+            ),
+        )

backend/tests/test_system_health.py

@@ -0,0 +1,304 @@
+import ast
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+from uuid import uuid4
+
+import psycopg
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from app.config import settings
+from app.identity import get_current_user
+from app.system_health.application import ApplicationMetadata
+from app.system_health.dependencies import get_system_health_service
+from app.system_health.frontend import router as frontend_router
+from app.system_health.repository import SystemHealthRepository
+from app.system_health.router import router as api_router
+from app.system_health.service import SystemHealthService
+
+
+APP_ROOT = Path(__file__).parents[1] / "app"
+EXPECTED_HEAD = "0014_repair_parts_tracking"
+
+
+def database_payload(revision=EXPECTED_HEAD):
+    return {
+        "migration": {"current_revision": revision},
+        "operations": {
+            "miners": 4, "sites": 1, "rooms": 2, "location_positions": 3,
+            "repairs": 1, "repair_events": 2, "repair_parts": 1,
+            "snapshot_count": 5,
+            "oldest_observed_at": datetime(2026, 8, 18, tzinfo=timezone.utc),
+            "latest_observed_at": datetime(2026, 8, 19, tzinfo=timezone.utc),
+            "hourly_present": True, "hourly_rows": 2,
+            "hourly_latest": datetime(2026, 8, 19, tzinfo=timezone.utc),
+            "daily_present": True, "daily_rows": 1,
+            "daily_latest": datetime(2026, 8, 19, tzinfo=timezone.utc).date(),
+            "retention_mode": "dry_run",
+            "retention_started_at": datetime(2026, 8, 19, tzinfo=timezone.utc),
+            "largest_tables": [{"name": "miners", "total_bytes": 8192}],
+        },
+        "integrity": {
+            "miners_without_asset_id": 1, "unmapped_miners": 1,
+            "miners_in_multiple_positions": 0, "duplicate_current_mac_groups": 0,
+            "unscoped_legacy_positions": 1, "expected_mac_mismatches": 1,
+            "occupied_positions": 2, "empty_positions": 1,
+            "active_modular_repairs": 1, "duplicate_active_repair_groups": 0,
+            "legacy_repair_only_miners": 0,
+            "lifecycle_repair_without_active_repair": 0,
+            "active_repair_lifecycle_mismatches": 0,
+            "waiting_placement_records": 1,
+        },
+    }
+
+
+class FakeRepository:
+    def __init__(self, payload=None, error=None):
+        self.payload = payload or database_payload()
+        self.error = error
+        self.calls = 0
+
+    def read_overview(self):
+        self.calls += 1
+        if self.error:
+            raise self.error
+        return self.payload
+
+
+class FakeRedis:
+    def __init__(self, available=True):
+        self.available = available
+        self.calls = 0
+
+    def ping(self):
+        self.calls += 1
+        return self.available
+
+
+def metadata(head=EXPECTED_HEAD):
+    return ApplicationMetadata(
+        environment="test", build_commit=None, build_version="0.1-test",
+        expected_migration_head=head,
+    )
+
+
+class SystemHealthServiceTests(unittest.TestCase):
+    def test_builds_typed_factual_overview_without_health_score(self):
+        repository, redis = FakeRepository(), FakeRedis()
+        result = SystemHealthService(repository, redis, metadata()).get_overview()
+        self.assertEqual(result.data.postgresql.migration.state, "current")
+        self.assertEqual(result.data.postgresql.telemetry.snapshot_count, 5)
+        self.assertEqual(result.data.postgresql.fleet.unmapped_miners, 1)
+        self.assertFalse(result.data.postgresql.retention.raw_retention_enabled)
+        self.assertEqual(result.data.postgresql.retention.hourly.population, "manual")
+        self.assertEqual(result.data.postgresql.retention.completeness, "not_guaranteed")
+        self.assertIsNone(result.data.application.build_commit)
+        self.assertFalse(result.meta.partial)
+        self.assertEqual((repository.calls, redis.calls), (1, 1))
+        self.assertNotIn("score", result.model_dump_json().lower())
+
+    def test_migration_mismatch_is_factual(self):
+        result = SystemHealthService(
+            FakeRepository(database_payload("0013_control_board_replacement")),
+            FakeRedis(), metadata(),
+        ).get_overview()
+        self.assertEqual(result.data.postgresql.migration.state, "mismatch")
+        self.assertEqual(
+            result.data.postgresql.migration.current_revision,
+            "0013_control_board_replacement",
+        )
+
+    def test_postgresql_failure_is_sanitized_and_redis_remains_available(self):
+        secret = "postgresql://user:password@private-host/minerops"
+        result = SystemHealthService(
+            FakeRepository(error=RuntimeError(secret)), FakeRedis(), metadata()
+        ).get_overview()
+        self.assertTrue(result.meta.partial)
+        self.assertEqual(result.meta.unavailable_sections, ["postgresql"])
+        self.assertEqual(result.data.postgresql.status, "unavailable")
+        self.assertEqual(result.data.redis.status, "available")
+        self.assertNotIn(secret, result.model_dump_json())
+
+    def test_redis_failure_preserves_database_diagnostics(self):
+        result = SystemHealthService(
+            FakeRepository(), FakeRedis(False), metadata()
+        ).get_overview()
+        self.assertEqual(result.data.postgresql.status, "available")
+        self.assertEqual(result.data.redis.status, "unavailable")
+        self.assertEqual(result.meta.unavailable_sections, ["redis"])
+
+
+class StubService:
+    def get_overview(self):
+        return SystemHealthService(
+            FakeRepository(), FakeRedis(), metadata()
+        ).get_overview()
+
+
+class SystemHealthRouterTests(unittest.TestCase):
+    def setUp(self):
+        self.app = FastAPI()
+        self.app.include_router(api_router)
+        self.app.dependency_overrides[get_system_health_service] = StubService
+        self.client = TestClient(self.app)
+
+    def tearDown(self):
+        self.client.close()
+
+    def authenticate(self, permissions):
+        self.app.dependency_overrides[get_current_user] = lambda: {
+            "id": str(uuid4()), "role": "Custom", "permissions": list(permissions)
+        }
+
+    def test_requires_authentication_and_system_health_permission(self):
+        self.assertEqual(self.client.get("/api/system/health-summary").status_code, 401)
+        self.authenticate(("dashboard.view",))
+        self.assertEqual(self.client.get("/api/system/health-summary").status_code, 403)
+
+    def test_returns_typed_no_store_envelope(self):
+        self.authenticate(("system_health.view",))
+        response = self.client.get("/api/system/health-summary")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.headers["cache-control"], "no-store")
+        payload = response.json()
+        self.assertEqual(payload["data"]["postgresql"]["status"], "available")
+        self.assertEqual(payload["meta"]["postgresql_statement_count"], 3)
+        self.assertIsNone(payload["error"])
+
+
+class CountingCursor(psycopg.Cursor):
+    statements = 0
+
+    def execute(self, query, params=None, *, prepare=None, binary=None):
+        type(self).statements += 1
+        return super().execute(
+            query, params, prepare=prepare, binary=binary
+        )
+
+
+class SystemHealthRepositoryIntegrationTests(unittest.TestCase):
+    def setUp(self):
+        self.connection = psycopg.connect(
+            host=settings.postgres_host, port=settings.postgres_port,
+            dbname=settings.postgres_db, user=settings.postgres_user,
+            password=settings.postgres_password, cursor_factory=CountingCursor,
+        )
+        CountingCursor.statements = 0
+        self.repository = SystemHealthRepository(connection=self.connection)
+
+    def tearDown(self):
+        self.connection.rollback()
+        self.connection.close()
+
+    def test_real_postgresql_overview_uses_exactly_three_statements(self):
+        site_id, room_id, mapped_id, unmapped_id = (
+            uuid4(), uuid4(), uuid4(), uuid4()
+        )
+        suffix = uuid4().hex[:10]
+        with self.connection.cursor() as cur:
+            cur.execute("INSERT INTO sites (id, name) VALUES (%s, %s)",
+                        (site_id, "Health Site " + suffix))
+            cur.execute("INSERT INTO rooms (id, site_id, name) VALUES (%s, %s, %s)",
+                        (room_id, site_id, "Health Room " + suffix))
+            cur.execute(
+                "INSERT INTO miners (id, asset_tag, lifecycle_status, current_mac) "
+                "VALUES (%s, %s, 'active', %s), (%s, NULL, 'active', NULL)",
+                (mapped_id, "HEALTH-" + suffix, "02:00:00:00:00:01", unmapped_id),
+            )
+            cur.execute(
+                "INSERT INTO location_positions "
+                "(room_id, room_name, rack_name, shelf_name, position_index, "
+                "miner_id, expected_mac) VALUES (%s, %s, %s, '1', 1, %s, %s), "
+                "(NULL, %s, %s, '1', 2, NULL, NULL)",
+                (room_id, "Health Room " + suffix, "Rack " + suffix, mapped_id,
+                 "02:00:00:00:00:02", "Legacy " + suffix, "Rack " + suffix),
+            )
+            cur.execute(
+                "INSERT INTO miner_status_snapshots (miner_id, observed_at, mhs_av) "
+                "VALUES (%s, %s, 0)",
+                (mapped_id, datetime(2026, 8, 19, 12, tzinfo=timezone.utc)),
+            )
+        CountingCursor.statements = 0
+        result = self.repository.read_overview()
+        self.assertEqual(CountingCursor.statements, 3)
+        self.assertGreaterEqual(result["operations"]["miners"], 2)
+        self.assertGreaterEqual(result["operations"]["snapshot_count"], 1)
+        self.assertEqual(result["migration"]["current_revision"], EXPECTED_HEAD)
+        self.assertGreaterEqual(result["integrity"]["unmapped_miners"], 1)
+        self.assertGreaterEqual(result["integrity"]["unscoped_legacy_positions"], 1)
+        self.assertGreaterEqual(result["integrity"]["expected_mac_mismatches"], 1)
+        self.assertTrue(result["operations"]["hourly_present"])
+        self.assertTrue(result["operations"]["daily_present"])
+
+    def test_existing_role_matrix_preserves_read_only_grant(self):
+        with self.connection.cursor() as cur:
+            cur.execute(
+                "SELECT roles.name FROM role_permissions "
+                "JOIN roles ON roles.id = role_permissions.role_id "
+                "JOIN permissions ON permissions.id = role_permissions.permission_id "
+                "WHERE permissions.key = 'system_health.view' ORDER BY roles.name"
+            )
+            roles = [row[0] for row in cur.fetchall()]
+        self.assertEqual(roles, ["Admin", "Manager", "Read Only"])
+
+
+class SystemHealthFrontendAndArchitectureTests(unittest.TestCase):
+    def test_page_is_semantic_safe_and_one_request(self):
+        app = FastAPI()
+        app.include_router(frontend_router)
+        app.dependency_overrides[get_current_user] = lambda: {
+            "id": str(uuid4()), "display_name": "Health <Operator>",
+            "role": "Manager", "permissions": ["system_health.view"],
+        }
+        client = TestClient(app)
+        response = client.get("/system-health")
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("System Health | MinerOps", response.text)
+        self.assertNotIn("Health <Operator>", response.text)
+        self.assertIn("aria-live=\"polite\"", response.text)
+        client.close()
+        script = (APP_ROOT / "frontend/static/system-health.js").read_text()
+        self.assertEqual(script.count('fetch("/api/system/health-summary"'), 1)
+        self.assertNotIn("setInterval", script)
+        self.assertNotIn("innerHTML", script)
+        self.assertIn("textContent", script)
+
+    def test_routes_are_unique_and_legacy_source_diagnostics_are_removed(self):
+        source = (APP_ROOT / "main.py").read_text()
+        self.assertNotIn("source_lines", source)
+        self.assertNotIn("recent_patch_markers", source)
+        from app.main import app
+        page = [r for r in app.routes if getattr(r, "path", None) == "/system-health"]
+        api = [r for r in app.routes if getattr(r, "path", None) == "/api/system/health-summary"]
+        self.assertEqual(len(page), 1)
+        self.assertEqual(len(api), 1)
+        self.assertEqual(page[0].methods, {"GET"})
+        self.assertEqual(api[0].methods, {"GET"})
+
+    def test_layers_and_security_boundaries(self):
+        router_source = (APP_ROOT / "system_health/router.py").read_text()
+        service_source = (APP_ROOT / "system_health/service.py").read_text()
+        repository_source = (APP_ROOT / "system_health/repository.py").read_text()
+        ast.parse(router_source)
+        ast.parse(service_source)
+        self.assertNotIn("SELECT ", router_source)
+        self.assertNotIn("SELECT ", service_source)
+        self.assertNotIn("fastapi", repository_source.lower())
+        self.assertNotIn("fastapi", service_source.lower())
+        static = "\n".join(
+            (APP_ROOT / path).read_text()
+            for path in (
+                "frontend/static/system-health.js",
+                "frontend/static/system-health.css",
+            )
+        ).lower()
+        for forbidden in (
+            "postgres_password", "redis_url", "jwt_secret", "connection string",
+            "stack trace", "password hash", "api key",
+        ):
+            self.assertNotIn(forbidden, static)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test-system-health.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${repo_root}"
+
+readonly source_files=(
+    backend/app/config.py
+    backend/app/frontend/static/system-health.css
+    backend/app/frontend/static/system-health.js
+    backend/app/frontend/system_health_view.py
+    backend/app/frontend/views.py
+    backend/app/main.py
+    backend/app/system_health/__init__.py
+    backend/app/system_health/adapters.py
+    backend/app/system_health/application.py
+    backend/app/system_health/dependencies.py
+    backend/app/system_health/frontend.py
+    backend/app/system_health/models.py
+    backend/app/system_health/repository.py
+    backend/app/system_health/router.py
+    backend/app/system_health/service.py
+    backend/tests/__init__.py
+    backend/tests/test_frontend_shell.py
+    backend/tests/test_system_health.py
+)
+
+for file in "${source_files[@]}"; do
+    [[ -r "${file}" ]] || { echo "Unreadable System Health input: ${file}" >&2; exit 1; }
+    [[ "$(stat -c '%a' "${file}")" == "644" ]] || {
+        echo "System Health source/test mode must be 0644: ${file}" >&2; exit 1;
+    }
+done
+[[ "$(stat -c '%a' scripts/test-system-health.sh)" == "755" ]] || {
+    echo "scripts/test-system-health.sh must have mode 0755." >&2; exit 1;
+}
+echo "System Health file-mode preflight passed."
+
+python3 -m py_compile backend/app/config.py backend/app/frontend/system_health_view.py backend/app/frontend/views.py backend/app/main.py backend/app/system_health/*.py backend/tests/__init__.py backend/tests/test_frontend_shell.py backend/tests/test_system_health.py
+node --check backend/app/frontend/static/system-health.js
+bash -n scripts/test-system-health.sh
+
+[[ -f .env.dev ]] || { echo "Missing .env.dev." >&2; exit 1; }
+project_name="${MINEROPS_SYSTEM_HEALTH_TEST_PROJECT:-minerops-system-health-$$}"
+compose=(docker compose --project-name "${project_name}" --env-file .env.dev -f compose.dev.yaml)
+cleanup() {
+    status=$?
+    if (( status != 0 )); then
+        "${compose[@]}" logs --no-color --tail 100 postgres redis || true
+    fi
+    "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
+    exit "${status}"
+}
+trap cleanup EXIT
+
+"${compose[@]}" up --detach postgres redis
+for attempt in $(seq 1 60); do
+    if "${compose[@]}" exec -T postgres sh -ec \
+        'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null 2>&1 \
+        && "${compose[@]}" exec -T redis redis-cli ping 2>/dev/null | grep -qx PONG; then
+        break
+    fi
+    (( attempt < 60 )) || { echo "Dependencies did not become ready." >&2; exit 1; }
+    sleep 1
+done
+"${compose[@]}" build backend
+"${compose[@]}" run --rm --no-deps backend alembic upgrade head
+"${compose[@]}" run --rm --no-deps \
+    -v "${repo_root}/backend/tests:/app/tests:ro" \
+    backend python -m unittest -v tests.test_system_health tests.test_frontend_shell