feat: add system health diagnostics

backend/app/frontend/static/system-health.css

@@ -0,0 +1,139 @@
+.health-toolbar,
+.health-section-head {
+    display: flex;
+    align-items: flex-start;
+    justify-content: space-between;
+    gap: 16px;
+}
+
+.health-toolbar {
+    margin-bottom: 12px;
+}
+
+.health-toolbar h2,
+.health-toolbar p,
+.health-section h2,
+.health-section h3 {
+    margin-top: 0;
+}
+
+.health-toolbar p {
+    margin-bottom: 0;
+    color: var(--muted-text, #52606d);
+}
+
+.health-notice {
+    border-left: 4px solid #64748b;
+    background: #f1f5f9;
+    margin: 0 0 14px;
+    padding: 10px 12px;
+}
+
+.health-notice[data-state="partial"] {
+    border-left-color: #b45309;
+    background: #fff7ed;
+}
+
+.health-notice[data-state="error"] {
+    border-left-color: #b91c1c;
+    background: #fef2f2;
+}
+
+.health-section {
+    border-top: 1px solid #cbd5e1;
+    padding: 16px 0;
+}
+
+.health-columns,
+.health-integrity-grid,
+.health-facts {
+    display: grid;
+    gap: 14px;
+}
+
+.health-columns {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+
+.health-integrity-grid {
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+}
+
+.health-facts {
+    grid-template-columns: repeat(4, minmax(0, 1fr));
+    margin: 0;
+}
+
+.health-facts-compact {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+
+.health-facts div {
+    min-width: 0;
+}
+
+.health-facts dt {
+    color: #52606d;
+    font-size: 0.78rem;
+    font-weight: 700;
+}
+
+.health-facts dd {
+    font-weight: 700;
+    margin: 3px 0 0;
+    overflow-wrap: anywhere;
+}
+
+.health-links {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 10px;
+}
+
+.health-links a:focus-visible,
+#healthRefresh:focus-visible {
+    outline: 3px solid #0f766e;
+    outline-offset: 3px;
+}
+
+.table-scroll {
+    overflow-x: auto;
+}
+
+.table-scroll table {
+    width: 100%;
+    border-collapse: collapse;
+}
+
+.table-scroll th,
+.table-scroll td {
+    border-bottom: 1px solid #dbe2e8;
+    padding: 7px 8px;
+    text-align: left;
+}
+
+@media (max-width: 900px) {
+    .health-integrity-grid,
+    .health-facts {
+        grid-template-columns: repeat(2, minmax(0, 1fr));
+    }
+}
+
+@media (max-width: 640px) {
+    .health-toolbar,
+    .health-section-head {
+        align-items: stretch;
+        flex-direction: column;
+    }
+
+    .health-columns,
+    .health-integrity-grid,
+    .health-facts,
+    .health-facts-compact {
+        grid-template-columns: 1fr;
+    }
+
+    #healthRefresh {
+        width: 100%;
+    }
+}

backend/app/frontend/static/system-health.js

@@ -0,0 +1,164 @@
+"use strict";
+
+(function () {
+    const notice = document.getElementById("healthNotice");
+    const content = document.getElementById("healthContent");
+    const refresh = document.getElementById("healthRefresh");
+
+    function setText(id, value, fallback) {
+        const element = document.getElementById(id);
+        if (element) {
+            element.textContent = value === null || value === undefined || value === ""
+                ? (fallback || "Unavailable") : String(value);
+        }
+    }
+
+    function number(value) {
+        return typeof value === "number" ? value.toLocaleString() : "Unavailable";
+    }
+
+    function timestamp(value) {
+        if (!value) return "Unavailable";
+        const date = new Date(value);
+        return Number.isNaN(date.valueOf()) ? "Unavailable" : date.toLocaleString();
+    }
+
+    function factualAge(value) {
+        if (!value) return "Unavailable";
+        const observed = new Date(value);
+        if (Number.isNaN(observed.valueOf())) return "Unavailable";
+        const seconds = Math.max(0, Math.floor((Date.now() - observed.valueOf()) / 1000));
+        if (seconds < 60) return seconds + " seconds ago";
+        const minutes = Math.floor(seconds / 60);
+        if (minutes < 60) return minutes + (minutes === 1 ? " minute ago" : " minutes ago");
+        const hours = Math.floor(minutes / 60);
+        if (hours < 24) return hours + (hours === 1 ? " hour ago" : " hours ago");
+        const days = Math.floor(hours / 24);
+        return days + (days === 1 ? " day ago" : " days ago");
+    }
+
+    function replaceFacts(id, facts) {
+        const list = document.getElementById(id);
+        list.replaceChildren();
+        for (const [label, value] of facts) {
+            const row = document.createElement("div");
+            const term = document.createElement("dt");
+            const detail = document.createElement("dd");
+            term.textContent = label;
+            detail.textContent = value;
+            row.append(term, detail);
+            list.append(row);
+        }
+    }
+
+    function render(payload) {
+        const data = payload.data;
+        const app = data.application;
+        const postgres = data.postgresql;
+        const migration = postgres.migration;
+        const telemetry = postgres.telemetry;
+        setText("healthAppStatus", app.status);
+        setText("healthEnvironment", app.environment);
+        setText("healthBuildCommit", app.build_commit);
+        setText("healthBuildVersion", app.build_version);
+        setText("healthServerTime", timestamp(app.server_time));
+        setText("healthMigrationState", migration.state);
+        setText("healthCurrentRevision", migration.current_revision);
+        setText("healthExpectedRevision", migration.expected_revision);
+        setText("healthPostgresqlStatus", postgres.status);
+        setText("healthRedisStatus", data.redis.status);
+        setText("healthSnapshotCount", telemetry ? number(telemetry.snapshot_count) : null);
+        setText("healthOldestObservation", telemetry ? timestamp(telemetry.oldest_observed_at) : null);
+        setText("healthLatestObservation", telemetry ? timestamp(telemetry.latest_observed_at) : null);
+        setText("healthObservationAge", telemetry ? factualAge(telemetry.latest_observed_at) : null);
+
+        const fleet = postgres.fleet;
+        replaceFacts("healthFleetFacts", fleet ? [
+            ["Total miners", number(fleet.total_miners)],
+            ["Without Asset ID", number(fleet.miners_without_asset_id)],
+            ["Unmapped", number(fleet.unmapped_miners)],
+            ["In multiple positions", number(fleet.miners_in_multiple_positions)],
+            ["Duplicate current MAC groups", number(fleet.duplicate_current_mac_groups)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const locations = postgres.locations;
+        replaceFacts("healthLocationFacts", locations ? [
+            ["Sites", number(locations.sites)], ["Rooms", number(locations.rooms)],
+            ["Positions", number(locations.positions)],
+            ["Occupied", number(locations.occupied_positions)],
+            ["Empty", number(locations.empty_positions)],
+            ["Legacy unscoped", number(locations.unscoped_legacy_positions)],
+            ["Expected MAC mismatches", number(locations.expected_mac_mismatches)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const repairs = postgres.repairs;
+        replaceFacts("healthRepairFacts", repairs ? [
+            ["Active modular repairs", number(repairs.active_modular_repairs)],
+            ["Duplicate active groups", number(repairs.duplicate_active_repair_groups)],
+            ["Legacy repair only", number(repairs.legacy_repair_only_miners)],
+            ["Lifecycle repair without active repair", number(repairs.lifecycle_repair_without_active_repair)],
+            ["Active repair lifecycle mismatches", number(repairs.active_repair_lifecycle_mismatches)],
+            ["Waiting placement", number(repairs.waiting_placement_records)],
+            ["Repair events", number(repairs.repair_events)],
+            ["Repair parts", number(repairs.repair_parts)]
+        ] : [["Diagnostics", "Unavailable"]]);
+        const retention = postgres.retention;
+        replaceFacts("healthRetentionFacts", retention ? [
+            ["Raw telemetry retention", retention.raw_retention_enabled ? "Enabled" : "Not enabled"],
+            ["Hourly rollups", retention.hourly.table_present ? "Present; manual population" : "Unavailable"],
+            ["Hourly latest bucket", timestamp(retention.hourly.latest_bucket)],
+            ["Daily rollups", retention.daily.table_present ? "Present; manual population" : "Unavailable"],
+            ["Daily latest bucket", timestamp(retention.daily.latest_bucket)],
+            ["Latest recorded run", timestamp(retention.latest_recorded_run_at)],
+            ["Latest run mode", retention.latest_recorded_run_mode || "None recorded"],
+            ["Completeness", "Not guaranteed"]
+        ] : [["Diagnostics", "Unavailable"]]);
+
+        const tableBody = document.getElementById("healthTableCounts");
+        tableBody.replaceChildren();
+        const rows = postgres.table_counts || [];
+        for (const table of rows) {
+            const row = document.createElement("tr");
+            const name = document.createElement("th");
+            const count = document.createElement("td");
+            name.scope = "row";
+            name.textContent = table.name;
+            count.textContent = number(table.row_count);
+            row.append(name, count);
+            tableBody.append(row);
+        }
+        if (!rows.length) {
+            const row = document.createElement("tr");
+            const cell = document.createElement("td");
+            cell.colSpan = 2;
+            cell.textContent = "Table diagnostics unavailable.";
+            row.append(cell);
+            tableBody.append(row);
+        }
+        content.hidden = false;
+        notice.dataset.state = payload.meta.partial ? "partial" : "ready";
+        notice.textContent = payload.meta.partial
+            ? "Diagnostics loaded with unavailable sections: " + payload.meta.unavailable_sections.join(", ") + "."
+            : "Diagnostics refreshed at " + timestamp(payload.meta.generated_at) + ".";
+    }
+
+    async function load() {
+        refresh.disabled = true;
+        notice.dataset.state = "loading";
+        notice.textContent = "Loading diagnostics.";
+        try {
+            const response = await fetch("/api/system/health-summary", {
+                credentials: "same-origin", headers: {"Accept": "application/json"}
+            });
+            const payload = await response.json().catch(function () { return {}; });
+            if (!response.ok) throw new Error("System Health request failed");
+            render(payload);
+        } catch (error) {
+            notice.dataset.state = "error";
+            notice.textContent = "System Health diagnostics could not be loaded.";
+        } finally {
+            refresh.disabled = false;
+        }
+    }
+
+    refresh.addEventListener("click", load);
+    load();
+}());