feat: add system health diagnostics

backend/app/frontend/static/system-health.css

@@ -0,0 +1,139 @@
+.health-toolbar,
+.health-section-head {
+    display: flex;
+    align-items: flex-start;
+    justify-content: space-between;
+    gap: 16px;
+}
+
+.health-toolbar {
+    margin-bottom: 12px;
+}
+
+.health-toolbar h2,
+.health-toolbar p,
+.health-section h2,
+.health-section h3 {
+    margin-top: 0;
+}
+
+.health-toolbar p {
+    margin-bottom: 0;
+    color: var(--muted-text, #52606d);
+}
+
+.health-notice {
+    border-left: 4px solid #64748b;
+    background: #f1f5f9;
+    margin: 0 0 14px;
+    padding: 10px 12px;
+}
+
+.health-notice[data-state="partial"] {
+    border-left-color: #b45309;
+    background: #fff7ed;
+}
+
+.health-notice[data-state="error"] {
+    border-left-color: #b91c1c;
+    background: #fef2f2;
+}
+
+.health-section {
+    border-top: 1px solid #cbd5e1;
+    padding: 16px 0;
+}
+
+.health-columns,
+.health-integrity-grid,
+.health-facts {
+    display: grid;
+    gap: 14px;
+}
+
+.health-columns {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+
+.health-integrity-grid {
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+}
+
+.health-facts {
+    grid-template-columns: repeat(4, minmax(0, 1fr));
+    margin: 0;
+}
+
+.health-facts-compact {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+
+.health-facts div {
+    min-width: 0;
+}
+
+.health-facts dt {
+    color: #52606d;
+    font-size: 0.78rem;
+    font-weight: 700;
+}
+
+.health-facts dd {
+    font-weight: 700;
+    margin: 3px 0 0;
+    overflow-wrap: anywhere;
+}
+
+.health-links {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 10px;
+}
+
+.health-links a:focus-visible,
+#healthRefresh:focus-visible {
+    outline: 3px solid #0f766e;
+    outline-offset: 3px;
+}
+
+.table-scroll {
+    overflow-x: auto;
+}
+
+.table-scroll table {
+    width: 100%;
+    border-collapse: collapse;
+}
+
+.table-scroll th,
+.table-scroll td {
+    border-bottom: 1px solid #dbe2e8;
+    padding: 7px 8px;
+    text-align: left;
+}
+
+@media (max-width: 900px) {
+    .health-integrity-grid,
+    .health-facts {
+        grid-template-columns: repeat(2, minmax(0, 1fr));
+    }
+}
+
+@media (max-width: 640px) {
+    .health-toolbar,
+    .health-section-head {
+        align-items: stretch;
+        flex-direction: column;
+    }
+
+    .health-columns,
+    .health-integrity-grid,
+    .health-facts,
+    .health-facts-compact {
+        grid-template-columns: 1fr;
+    }
+
+    #healthRefresh {
+        width: 100%;
+    }
+}

backend/app/frontend/static/system-health.js

@@ -0,0 +1,164 @@
+"use strict";
+
+(function () {
+    const notice = document.getElementById("healthNotice");
+    const content = document.getElementById("healthContent");
+    const refresh = document.getElementById("healthRefresh");
+
+    function setText(id, value, fallback) {
+        const element = document.getElementById(id);
+        if (element) {
+            element.textContent = value === null || value === undefined || value === ""
+                ? (fallback || "Unavailable") : String(value);
+        }
+    }
+
+    function number(value) {
+        return typeof value === "number" ? value.toLocaleString() : "Unavailable";
+    }
+
+    function timestamp(value) {
+        if (!value) return "Unavailable";
+        const date = new Date(value);
+        return Number.isNaN(date.valueOf()) ? "Unavailable" : date.toLocaleString();
+    }
+
+    function factualAge(value) {
+        if (!value) return "Unavailable";
+        const observed = new Date(value);
+        if (Number.isNaN(observed.valueOf())) return "Unavailable";
+        const seconds = Math.max(0, Math.floor((Date.now() - observed.valueOf()) / 1000));
+        if (seconds < 60) return seconds + " seconds ago";
+        const minutes = Math.floor(seconds / 60);
+        if (minutes < 60) return minutes + (minutes === 1 ? " minute ago" : " minutes ago");
+        const hours = Math.floor(minutes / 60);
+        if (hours < 24) return hours + (hours === 1 ? " hour ago" : " hours ago");
+        const days = Math.floor(hours / 24);
+        return days + (days === 1 ? " day ago" : " days ago");
+    }
+
+    function replaceFacts(id, facts) {
+        const list = document.getElementById(id);
+        list.replaceChildren();
+        for (const [label, value] of facts) {
+            const row = document.createElement("div");
+            const term = document.createElement("dt");
+            const detail = document.createElement("dd");
+            term.textContent = label;
+            detail.textContent = value;
+            row.append(term, detail);
+            list.append(row);
+        }
+    }
+
+    function render(payload) {
+        const data = payload.data;
+        const app = data.application;
+        const postgres = data.postgresql;
+        const migration = postgres.migration;
+        const telemetry = postgres.telemetry;
+        setText("healthAppStatus", app.status);
+        setText("healthEnvironment", app.environment);
+        setText("healthBuildCommit", app.build_commit);
+        setText("healthBuildVersion", app.build_version);
+        setText("healthServerTime", timestamp(app.server_time));
+        setText("healthMigrationState", migration.state);
+        setText("healthCurrentRevision", migration.current_revision);
+        setText("healthExpectedRevision", migration.expected_revision);
+        setText("healthPostgresqlStatus", postgres.status);
+        setText("healthRedisStatus", data.redis.status);
+        setText("healthSnapshotCount", telemetry ? number(telemetry.snapshot_count) : null);
+        setText("healthOldestObservation", telemetry ? timestamp(telemetry.oldest_observed_at) : null);
+        setText("healthLatestObservation", telemetry ? timestamp(telemetry.latest_observed_at) : null);
+        setText("healthObservationAge", telemetry ? factualAge(telemetry.latest_observed_at) : null);
+
+        const fleet = postgres.fleet;
+        replaceFacts("healthFleetFacts", fleet ? [
+            ["Total miners", number(fleet.total_miners)],
+            ["Without Asset ID", number(fleet.miners_without_asset_id)],
+            ["Unmapped", number(fleet.unmapped_miners)],
+            ["In multiple positions", number(fleet.miners_in_multiple_positions)],
+            ["Duplicate current MAC groups", number(fleet.duplicate_current_mac_groups)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const locations = postgres.locations;
+        replaceFacts("healthLocationFacts", locations ? [
+            ["Sites", number(locations.sites)], ["Rooms", number(locations.rooms)],
+            ["Positions", number(locations.positions)],
+            ["Occupied", number(locations.occupied_positions)],
+            ["Empty", number(locations.empty_positions)],
+            ["Legacy unscoped", number(locations.unscoped_legacy_positions)],
+            ["Expected MAC mismatches", number(locations.expected_mac_mismatches)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const repairs = postgres.repairs;
+        replaceFacts("healthRepairFacts", repairs ? [
+            ["Active modular repairs", number(repairs.active_modular_repairs)],
+            ["Duplicate active groups", number(repairs.duplicate_active_repair_groups)],
+            ["Legacy repair only", number(repairs.legacy_repair_only_miners)],
+            ["Lifecycle repair without active repair", number(repairs.lifecycle_repair_without_active_repair)],
+            ["Active repair lifecycle mismatches", number(repairs.active_repair_lifecycle_mismatches)],
+            ["Waiting placement", number(repairs.waiting_placement_records)],
+            ["Repair events", number(repairs.repair_events)],
+            ["Repair parts", number(repairs.repair_parts)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const retention = postgres.retention;
+        replaceFacts("healthRetentionFacts", retention ? [
+            ["Raw telemetry retention", retention.raw_retention_enabled ? "Enabled" : "Not enabled"],
+            ["Hourly rollups", retention.hourly.table_present ? "Present; manual population" : "Unavailable"],
+            ["Hourly latest bucket", timestamp(retention.hourly.latest_bucket)],
+            ["Daily rollups", retention.daily.table_present ? "Present; manual population" : "Unavailable"],
+            ["Daily latest bucket", timestamp(retention.daily.latest_bucket)],
+            ["Latest recorded run", timestamp(retention.latest_recorded_run_at)],
+            ["Latest run mode", retention.latest_recorded_run_mode || "None recorded"],
+            ["Completeness", "Not guaranteed"]
+        ] : [["Diagnostics", "Unavailable"]]);
+
+        const tableBody = document.getElementById("healthTableCounts");
+        tableBody.replaceChildren();
+        const rows = postgres.table_counts || [];
+        for (const table of rows) {
+            const row = document.createElement("tr");
+            const name = document.createElement("th");
+            const count = document.createElement("td");
+            name.scope = "row";
+            name.textContent = table.name;
+            count.textContent = number(table.row_count);
+            row.append(name, count);
+            tableBody.append(row);
+        }
+        if (!rows.length) {
+            const row = document.createElement("tr");
+            const cell = document.createElement("td");
+            cell.colSpan = 2;
+            cell.textContent = "Table diagnostics unavailable.";
+            row.append(cell);
+            tableBody.append(row);
+        }
+        content.hidden = false;
+        notice.dataset.state = payload.meta.partial ? "partial" : "ready";
+        notice.textContent = payload.meta.partial
+            ? "Diagnostics loaded with unavailable sections: " + payload.meta.unavailable_sections.join(", ") + "."
+            : "Diagnostics refreshed at " + timestamp(payload.meta.generated_at) + ".";
+    }
+
+    async function load() {
+        refresh.disabled = true;
+        notice.dataset.state = "loading";
+        notice.textContent = "Loading diagnostics.";
+        try {
+            const response = await fetch("/api/system/health-summary", {
+                credentials: "same-origin", headers: {"Accept": "application/json"}
+            });
+            const payload = await response.json().catch(function () { return {}; });
+            if (!response.ok) throw new Error("System Health request failed");
+            render(payload);
+        } catch (error) {
+            notice.dataset.state = "error";
+            notice.textContent = "System Health diagnostics could not be loaded.";
+        } finally {
+            refresh.disabled = false;
+        }
+    }
+
+    refresh.addEventListener("click", load);
+    load();
+}());

backend/app/frontend/system_health_view.py

@@ -0,0 +1,54 @@
+"""Semantic content for the Internal Alpha System Health page."""
+
+
+def system_health_content() -> str:
+    return """
+<section class="health-toolbar" aria-labelledby="healthOverviewTitle">
+    <div><h2 id="healthOverviewTitle">Operations diagnostics</h2>
+        <p>Availability, data facts, integrity counts, and maintenance state.</p></div>
+    <button class="primary-button" id="healthRefresh" type="button">Refresh</button>
+</section>
+<p class="health-notice" id="healthNotice" role="status" aria-live="polite">Loading diagnostics.</p>
+<div id="healthContent" hidden>
+    <section class="health-section" aria-labelledby="applicationHealthTitle">
+        <h2 id="applicationHealthTitle">Application</h2>
+        <dl class="health-facts">
+            <div><dt>Application</dt><dd id="healthAppStatus">-</dd></div>
+            <div><dt>Environment</dt><dd id="healthEnvironment">-</dd></div>
+            <div><dt>Build commit</dt><dd id="healthBuildCommit">-</dd></div>
+            <div><dt>Build version</dt><dd id="healthBuildVersion">-</dd></div>
+            <div><dt>Server time</dt><dd id="healthServerTime">-</dd></div>
+            <div><dt>Schema</dt><dd id="healthMigrationState">-</dd></div>
+            <div><dt>Database revision</dt><dd id="healthCurrentRevision">-</dd></div>
+            <div><dt>Expected revision</dt><dd id="healthExpectedRevision">-</dd></div>
+        </dl>
+    </section>
+    <div class="health-columns">
+        <section class="health-section" aria-labelledby="servicesHealthTitle"><h2 id="servicesHealthTitle">Service availability</h2>
+            <dl class="health-facts health-facts-compact"><div><dt>PostgreSQL</dt><dd id="healthPostgresqlStatus">-</dd></div>
+                <div><dt>Redis</dt><dd id="healthRedisStatus">-</dd></div></dl></section>
+        <section class="health-section" aria-labelledby="telemetryHealthTitle"><h2 id="telemetryHealthTitle">Telemetry</h2>
+            <dl class="health-facts health-facts-compact"><div><dt>Raw snapshots</dt><dd id="healthSnapshotCount">-</dd></div>
+                <div><dt>Oldest observation</dt><dd id="healthOldestObservation">-</dd></div>
+                <div><dt>Latest observation</dt><dd id="healthLatestObservation">-</dd></div>
+                <div><dt>Latest age</dt><dd id="healthObservationAge">-</dd></div></dl></section>
+    </div>
+    <section class="health-section" aria-labelledby="integrityHealthTitle">
+        <div class="health-section-head"><h2 id="integrityHealthTitle">Operational integrity facts</h2>
+            <div class="health-links"><a href="/assets/miners">Fleet</a><a href="/monitoring/map">Mine Map</a>
+                <a href="/operations/repairs">Repair Center</a><a href="/monitoring/analytics">Trends</a></div></div>
+        <div class="health-integrity-grid">
+            <section aria-labelledby="fleetIntegrityTitle"><h3 id="fleetIntegrityTitle">Fleet</h3><dl id="healthFleetFacts" class="health-facts health-facts-compact"></dl></section>
+            <section aria-labelledby="locationIntegrityTitle"><h3 id="locationIntegrityTitle">Locations</h3><dl id="healthLocationFacts" class="health-facts health-facts-compact"></dl></section>
+            <section aria-labelledby="repairIntegrityTitle"><h3 id="repairIntegrityTitle">Repairs</h3><dl id="healthRepairFacts" class="health-facts health-facts-compact"></dl></section>
+        </div>
+    </section>
+    <div class="health-columns">
+        <section class="health-section" aria-labelledby="retentionHealthTitle"><h2 id="retentionHealthTitle">Retention and rollups</h2>
+            <dl id="healthRetentionFacts" class="health-facts health-facts-compact"></dl></section>
+        <section class="health-section" aria-labelledby="tablesHealthTitle"><h2 id="tablesHealthTitle">Operational tables</h2>
+            <div class="table-scroll"><table><thead><tr><th scope="col">Table</th><th scope="col">Rows</th></tr></thead>
+                <tbody id="healthTableCounts"></tbody></table></div></section>
+    </div>
+</div>
+"""

backend/app/frontend/views.py

@@ -5,6 +5,7 @@ from typing import Callable
 
 from app.frontend.miner_detail_view import miner_detail_content
 from app.frontend.monitoring_trends_view import monitoring_trends_content
+from app.frontend.system_health_view import system_health_content
 
 
 NAVIGATION = (
@@ -13,6 +14,7 @@ NAVIGATION = (
     ("analytics", "Trends", "/monitoring/analytics", "miners.view"),
     ("repairs", "Repair Center", "/operations/repairs", "repairs.view"),
     ("miners", "Miners", "/assets/miners", "miners.view"),
+    ("system_health", "System Health", "/system-health", "system_health.view"),
 )
 
 PAGE_DETAILS = {
@@ -46,6 +48,11 @@ PAGE_DETAILS = {
         "Miner Detail",
         "Current identity, operating facts, location, and authorized history.",
     ),
+    "system_health": (
+        "Administration / Diagnostics",
+        "System Health",
+        "Inspect application services, operational data, and maintenance facts.",
+    ),
 }
 
 
@@ -455,7 +462,9 @@ def render_app_page(
         else '<div class="global-search-unavailable" aria-hidden="true"></div>'
     )
     content = (
-        miner_detail_content()
+        system_health_content()
+        if view == "system_health"
+        else miner_detail_content()
         if view == "miner_detail"
         else monitoring_trends_content(has_permission("locations.view"))
         if view == "analytics"
@@ -474,7 +483,9 @@ def render_app_page(
         )
     )
     page_script = (
-        '<script src="/assets/minerops-miner-detail.js" defer></script>'
+        '<script src="/assets/minerops-system-health.js" defer></script>'
+        if view == "system_health"
+        else '<script src="/assets/minerops-miner-detail.js" defer></script>'
         if view == "miner_detail"
         else '<script src="/assets/minerops-monitoring-trends.js" defer></script>'
         if view == "analytics"
@@ -489,7 +500,9 @@ def render_app_page(
         else ""
     )
     page_style = (
-        '<link rel="stylesheet" href="/assets/minerops-monitoring-trends.css">'
+        '<link rel="stylesheet" href="/assets/minerops-system-health.css">'
+        if view == "system_health"
+        else '<link rel="stylesheet" href="/assets/minerops-monitoring-trends.css">'
         if view == "analytics" else ""
     )
     object_attributes = ""