feat: add system health diagnostics
This commit is contained in:
lussierj committed
1 parent
0dd26a3f24
commit
c3501b987e
17 files changed
+1335
-741
No files matched your search
@@ -5,6 +5,8 @@ class Settings(BaseSettings):
|
||||
app_name: str = "MinerOps Console"
|
||||
app_env: str = "development"
|
||||
app_timezone: str = "America/Winnipeg"
|
||||
minerops_build_commit: str | None = None
|
||||
minerops_build_version: str | None = None
|
||||
|
||||
postgres_db: str = "minerops"
|
||||
postgres_user: str = "minerops"
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
.health-toolbar,
|
||||
.health-section-head {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.health-toolbar {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.health-toolbar h2,
|
||||
.health-toolbar p,
|
||||
.health-section h2,
|
||||
.health-section h3 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.health-toolbar p {
|
||||
margin-bottom: 0;
|
||||
color: var(--muted-text, #52606d);
|
||||
}
|
||||
|
||||
.health-notice {
|
||||
border-left: 4px solid #64748b;
|
||||
background: #f1f5f9;
|
||||
margin: 0 0 14px;
|
||||
padding: 10px 12px;
|
||||
}
|
||||
|
||||
.health-notice[data-state="partial"] {
|
||||
border-left-color: #b45309;
|
||||
background: #fff7ed;
|
||||
}
|
||||
|
||||
.health-notice[data-state="error"] {
|
||||
border-left-color: #b91c1c;
|
||||
background: #fef2f2;
|
||||
}
|
||||
|
||||
.health-section {
|
||||
border-top: 1px solid #cbd5e1;
|
||||
padding: 16px 0;
|
||||
}
|
||||
|
||||
.health-columns,
|
||||
.health-integrity-grid,
|
||||
.health-facts {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.health-columns {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.health-integrity-grid {
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.health-facts {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.health-facts-compact {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.health-facts div {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.health-facts dt {
|
||||
color: #52606d;
|
||||
font-size: 0.78rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.health-facts dd {
|
||||
font-weight: 700;
|
||||
margin: 3px 0 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.health-links {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.health-links a:focus-visible,
|
||||
#healthRefresh:focus-visible {
|
||||
outline: 3px solid #0f766e;
|
||||
outline-offset: 3px;
|
||||
}
|
||||
|
||||
.table-scroll {
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.table-scroll table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.table-scroll th,
|
||||
.table-scroll td {
|
||||
border-bottom: 1px solid #dbe2e8;
|
||||
padding: 7px 8px;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.health-integrity-grid,
|
||||
.health-facts {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.health-toolbar,
|
||||
.health-section-head {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.health-columns,
|
||||
.health-integrity-grid,
|
||||
.health-facts,
|
||||
.health-facts-compact {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
#healthRefresh {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
"use strict";
|
||||
|
||||
(function () {
|
||||
const notice = document.getElementById("healthNotice");
|
||||
const content = document.getElementById("healthContent");
|
||||
const refresh = document.getElementById("healthRefresh");
|
||||
|
||||
function setText(id, value, fallback) {
|
||||
const element = document.getElementById(id);
|
||||
if (element) {
|
||||
element.textContent = value === null || value === undefined || value === ""
|
||||
? (fallback || "Unavailable") : String(value);
|
||||
}
|
||||
}
|
||||
|
||||
function number(value) {
|
||||
return typeof value === "number" ? value.toLocaleString() : "Unavailable";
|
||||
}
|
||||
|
||||
function timestamp(value) {
|
||||
if (!value) return "Unavailable";
|
||||
const date = new Date(value);
|
||||
return Number.isNaN(date.valueOf()) ? "Unavailable" : date.toLocaleString();
|
||||
}
|
||||
|
||||
function factualAge(value) {
|
||||
if (!value) return "Unavailable";
|
||||
const observed = new Date(value);
|
||||
if (Number.isNaN(observed.valueOf())) return "Unavailable";
|
||||
const seconds = Math.max(0, Math.floor((Date.now() - observed.valueOf()) / 1000));
|
||||
if (seconds < 60) return seconds + " seconds ago";
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
if (minutes < 60) return minutes + (minutes === 1 ? " minute ago" : " minutes ago");
|
||||
const hours = Math.floor(minutes / 60);
|
||||
if (hours < 24) return hours + (hours === 1 ? " hour ago" : " hours ago");
|
||||
const days = Math.floor(hours / 24);
|
||||
return days + (days === 1 ? " day ago" : " days ago");
|
||||
}
|
||||
|
||||
function replaceFacts(id, facts) {
|
||||
const list = document.getElementById(id);
|
||||
list.replaceChildren();
|
||||
for (const [label, value] of facts) {
|
||||
const row = document.createElement("div");
|
||||
const term = document.createElement("dt");
|
||||
const detail = document.createElement("dd");
|
||||
term.textContent = label;
|
||||
detail.textContent = value;
|
||||
row.append(term, detail);
|
||||
list.append(row);
|
||||
}
|
||||
}
|
||||
|
||||
function render(payload) {
|
||||
const data = payload.data;
|
||||
const app = data.application;
|
||||
const postgres = data.postgresql;
|
||||
const migration = postgres.migration;
|
||||
const telemetry = postgres.telemetry;
|
||||
setText("healthAppStatus", app.status);
|
||||
setText("healthEnvironment", app.environment);
|
||||
setText("healthBuildCommit", app.build_commit);
|
||||
setText("healthBuildVersion", app.build_version);
|
||||
setText("healthServerTime", timestamp(app.server_time));
|
||||
setText("healthMigrationState", migration.state);
|
||||
setText("healthCurrentRevision", migration.current_revision);
|
||||
setText("healthExpectedRevision", migration.expected_revision);
|
||||
setText("healthPostgresqlStatus", postgres.status);
|
||||
setText("healthRedisStatus", data.redis.status);
|
||||
setText("healthSnapshotCount", telemetry ? number(telemetry.snapshot_count) : null);
|
||||
setText("healthOldestObservation", telemetry ? timestamp(telemetry.oldest_observed_at) : null);
|
||||
setText("healthLatestObservation", telemetry ? timestamp(telemetry.latest_observed_at) : null);
|
||||
setText("healthObservationAge", telemetry ? factualAge(telemetry.latest_observed_at) : null);
|
||||
|
||||
const fleet = postgres.fleet;
|
||||
replaceFacts("healthFleetFacts", fleet ? [
|
||||
["Total miners", number(fleet.total_miners)],
|
||||
["Without Asset ID", number(fleet.miners_without_asset_id)],
|
||||
["Unmapped", number(fleet.unmapped_miners)],
|
||||
["In multiple positions", number(fleet.miners_in_multiple_positions)],
|
||||
["Duplicate current MAC groups", number(fleet.duplicate_current_mac_groups)]
|
||||
] : [["Diagnostics", "Unavailable"]]);
|
||||
const locations = postgres.locations;
|
||||
replaceFacts("healthLocationFacts", locations ? [
|
||||
["Sites", number(locations.sites)], ["Rooms", number(locations.rooms)],
|
||||
["Positions", number(locations.positions)],
|
||||
["Occupied", number(locations.occupied_positions)],
|
||||
["Empty", number(locations.empty_positions)],
|
||||
["Legacy unscoped", number(locations.unscoped_legacy_positions)],
|
||||
["Expected MAC mismatches", number(locations.expected_mac_mismatches)]
|
||||
] : [["Diagnostics", "Unavailable"]]);
|
||||
const repairs = postgres.repairs;
|
||||
replaceFacts("healthRepairFacts", repairs ? [
|
||||
["Active modular repairs", number(repairs.active_modular_repairs)],
|
||||
["Duplicate active groups", number(repairs.duplicate_active_repair_groups)],
|
||||
["Legacy repair only", number(repairs.legacy_repair_only_miners)],
|
||||
["Lifecycle repair without active repair", number(repairs.lifecycle_repair_without_active_repair)],
|
||||
["Active repair lifecycle mismatches", number(repairs.active_repair_lifecycle_mismatches)],
|
||||
["Waiting placement", number(repairs.waiting_placement_records)],
|
||||
["Repair events", number(repairs.repair_events)],
|
||||
["Repair parts", number(repairs.repair_parts)]
|
||||
] : [["Diagnostics", "Unavailable"]]);
|
||||
const retention = postgres.retention;
|
||||
replaceFacts("healthRetentionFacts", retention ? [
|
||||
["Raw telemetry retention", retention.raw_retention_enabled ? "Enabled" : "Not enabled"],
|
||||
["Hourly rollups", retention.hourly.table_present ? "Present; manual population" : "Unavailable"],
|
||||
["Hourly latest bucket", timestamp(retention.hourly.latest_bucket)],
|
||||
["Daily rollups", retention.daily.table_present ? "Present; manual population" : "Unavailable"],
|
||||
["Daily latest bucket", timestamp(retention.daily.latest_bucket)],
|
||||
["Latest recorded run", timestamp(retention.latest_recorded_run_at)],
|
||||
["Latest run mode", retention.latest_recorded_run_mode || "None recorded"],
|
||||
["Completeness", "Not guaranteed"]
|
||||
] : [["Diagnostics", "Unavailable"]]);
|
||||
|
||||
const tableBody = document.getElementById("healthTableCounts");
|
||||
tableBody.replaceChildren();
|
||||
const rows = postgres.table_counts || [];
|
||||
for (const table of rows) {
|
||||
const row = document.createElement("tr");
|
||||
const name = document.createElement("th");
|
||||
const count = document.createElement("td");
|
||||
name.scope = "row";
|
||||
name.textContent = table.name;
|
||||
count.textContent = number(table.row_count);
|
||||
row.append(name, count);
|
||||
tableBody.append(row);
|
||||
}
|
||||
if (!rows.length) {
|
||||
const row = document.createElement("tr");
|
||||
const cell = document.createElement("td");
|
||||
cell.colSpan = 2;
|
||||
cell.textContent = "Table diagnostics unavailable.";
|
||||
row.append(cell);
|
||||
tableBody.append(row);
|
||||
}
|
||||
content.hidden = false;
|
||||
notice.dataset.state = payload.meta.partial ? "partial" : "ready";
|
||||
notice.textContent = payload.meta.partial
|
||||
? "Diagnostics loaded with unavailable sections: " + payload.meta.unavailable_sections.join(", ") + "."
|
||||
: "Diagnostics refreshed at " + timestamp(payload.meta.generated_at) + ".";
|
||||
}
|
||||
|
||||
async function load() {
|
||||
refresh.disabled = true;
|
||||
notice.dataset.state = "loading";
|
||||
notice.textContent = "Loading diagnostics.";
|
||||
try {
|
||||
const response = await fetch("/api/system/health-summary", {
|
||||
credentials: "same-origin", headers: {"Accept": "application/json"}
|
||||
});
|
||||
const payload = await response.json().catch(function () { return {}; });
|
||||
if (!response.ok) throw new Error("System Health request failed");
|
||||
render(payload);
|
||||
} catch (error) {
|
||||
notice.dataset.state = "error";
|
||||
notice.textContent = "System Health diagnostics could not be loaded.";
|
||||
} finally {
|
||||
refresh.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
refresh.addEventListener("click", load);
|
||||
load();
|
||||
}());
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Semantic content for the Internal Alpha System Health page."""
|
||||
|
||||
|
||||
def system_health_content() -> str:
|
||||
return """
|
||||
<section class="health-toolbar" aria-labelledby="healthOverviewTitle">
|
||||
<div><h2 id="healthOverviewTitle">Operations diagnostics</h2>
|
||||
<p>Availability, data facts, integrity counts, and maintenance state.</p></div>
|
||||
<button class="primary-button" id="healthRefresh" type="button">Refresh</button>
|
||||
</section>
|
||||
<p class="health-notice" id="healthNotice" role="status" aria-live="polite">Loading diagnostics.</p>
|
||||
<div id="healthContent" hidden>
|
||||
<section class="health-section" aria-labelledby="applicationHealthTitle">
|
||||
<h2 id="applicationHealthTitle">Application</h2>
|
||||
<dl class="health-facts">
|
||||
<div><dt>Application</dt><dd id="healthAppStatus">-</dd></div>
|
||||
<div><dt>Environment</dt><dd id="healthEnvironment">-</dd></div>
|
||||
<div><dt>Build commit</dt><dd id="healthBuildCommit">-</dd></div>
|
||||
<div><dt>Build version</dt><dd id="healthBuildVersion">-</dd></div>
|
||||
<div><dt>Server time</dt><dd id="healthServerTime">-</dd></div>
|
||||
<div><dt>Schema</dt><dd id="healthMigrationState">-</dd></div>
|
||||
<div><dt>Database revision</dt><dd id="healthCurrentRevision">-</dd></div>
|
||||
<div><dt>Expected revision</dt><dd id="healthExpectedRevision">-</dd></div>
|
||||
</dl>
|
||||
</section>
|
||||
<div class="health-columns">
|
||||
<section class="health-section" aria-labelledby="servicesHealthTitle"><h2 id="servicesHealthTitle">Service availability</h2>
|
||||
<dl class="health-facts health-facts-compact"><div><dt>PostgreSQL</dt><dd id="healthPostgresqlStatus">-</dd></div>
|
||||
<div><dt>Redis</dt><dd id="healthRedisStatus">-</dd></div></dl></section>
|
||||
<section class="health-section" aria-labelledby="telemetryHealthTitle"><h2 id="telemetryHealthTitle">Telemetry</h2>
|
||||
<dl class="health-facts health-facts-compact"><div><dt>Raw snapshots</dt><dd id="healthSnapshotCount">-</dd></div>
|
||||
<div><dt>Oldest observation</dt><dd id="healthOldestObservation">-</dd></div>
|
||||
<div><dt>Latest observation</dt><dd id="healthLatestObservation">-</dd></div>
|
||||
<div><dt>Latest age</dt><dd id="healthObservationAge">-</dd></div></dl></section>
|
||||
</div>
|
||||
<section class="health-section" aria-labelledby="integrityHealthTitle">
|
||||
<div class="health-section-head"><h2 id="integrityHealthTitle">Operational integrity facts</h2>
|
||||
<div class="health-links"><a href="/assets/miners">Fleet</a><a href="/monitoring/map">Mine Map</a>
|
||||
<a href="/operations/repairs">Repair Center</a><a href="/monitoring/analytics">Trends</a></div></div>
|
||||
<div class="health-integrity-grid">
|
||||
<section aria-labelledby="fleetIntegrityTitle"><h3 id="fleetIntegrityTitle">Fleet</h3><dl id="healthFleetFacts" class="health-facts health-facts-compact"></dl></section>
|
||||
<section aria-labelledby="locationIntegrityTitle"><h3 id="locationIntegrityTitle">Locations</h3><dl id="healthLocationFacts" class="health-facts health-facts-compact"></dl></section>
|
||||
<section aria-labelledby="repairIntegrityTitle"><h3 id="repairIntegrityTitle">Repairs</h3><dl id="healthRepairFacts" class="health-facts health-facts-compact"></dl></section>
|
||||
</div>
|
||||
</section>
|
||||
<div class="health-columns">
|
||||
<section class="health-section" aria-labelledby="retentionHealthTitle"><h2 id="retentionHealthTitle">Retention and rollups</h2>
|
||||
<dl id="healthRetentionFacts" class="health-facts health-facts-compact"></dl></section>
|
||||
<section class="health-section" aria-labelledby="tablesHealthTitle"><h2 id="tablesHealthTitle">Operational tables</h2>
|
||||
<div class="table-scroll"><table><thead><tr><th scope="col">Table</th><th scope="col">Rows</th></tr></thead>
|
||||
<tbody id="healthTableCounts"></tbody></table></div></section>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
@@ -5,6 +5,7 @@ from typing import Callable
|
||||
|
||||
from app.frontend.miner_detail_view import miner_detail_content
|
||||
from app.frontend.monitoring_trends_view import monitoring_trends_content
|
||||
from app.frontend.system_health_view import system_health_content
|
||||
|
||||
|
||||
NAVIGATION = (
|
||||
@@ -13,6 +14,7 @@ NAVIGATION = (
|
||||
("analytics", "Trends", "/monitoring/analytics", "miners.view"),
|
||||
("repairs", "Repair Center", "/operations/repairs", "repairs.view"),
|
||||
("miners", "Miners", "/assets/miners", "miners.view"),
|
||||
("system_health", "System Health", "/system-health", "system_health.view"),
|
||||
)
|
||||
|
||||
PAGE_DETAILS = {
|
||||
@@ -46,6 +48,11 @@ PAGE_DETAILS = {
|
||||
"Miner Detail",
|
||||
"Current identity, operating facts, location, and authorized history.",
|
||||
),
|
||||
"system_health": (
|
||||
"Administration / Diagnostics",
|
||||
"System Health",
|
||||
"Inspect application services, operational data, and maintenance facts.",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -455,7 +462,9 @@ def render_app_page(
|
||||
else '<div class="global-search-unavailable" aria-hidden="true"></div>'
|
||||
)
|
||||
content = (
|
||||
miner_detail_content()
|
||||
system_health_content()
|
||||
if view == "system_health"
|
||||
else miner_detail_content()
|
||||
if view == "miner_detail"
|
||||
else monitoring_trends_content(has_permission("locations.view"))
|
||||
if view == "analytics"
|
||||
@@ -474,7 +483,9 @@ def render_app_page(
|
||||
)
|
||||
)
|
||||
page_script = (
|
||||
'<script src="/assets/minerops-miner-detail.js" defer></script>'
|
||||
'<script src="/assets/minerops-system-health.js" defer></script>'
|
||||
if view == "system_health"
|
||||
else '<script src="/assets/minerops-miner-detail.js" defer></script>'
|
||||
if view == "miner_detail"
|
||||
else '<script src="/assets/minerops-monitoring-trends.js" defer></script>'
|
||||
if view == "analytics"
|
||||
@@ -489,7 +500,9 @@ def render_app_page(
|
||||
else ""
|
||||
)
|
||||
page_style = (
|
||||
'<link rel="stylesheet" href="/assets/minerops-monitoring-trends.css">'
|
||||
'<link rel="stylesheet" href="/assets/minerops-system-health.css">'
|
||||
if view == "system_health"
|
||||
else '<link rel="stylesheet" href="/assets/minerops-monitoring-trends.css">'
|
||||
if view == "analytics" else ""
|
||||
)
|
||||
object_attributes = ""
|
||||
|
||||
+4
-738
@@ -32,6 +32,8 @@ from app.components.router import router as components_router
|
||||
from app.miner_detail.router import router as miner_detail_router
|
||||
from app.monitoring.router import router as monitoring_router
|
||||
from app.monitoring.frontend import router as monitoring_frontend_router
|
||||
from app.system_health.frontend import router as system_health_frontend_router
|
||||
from app.system_health.router import router as system_health_router
|
||||
from app.repairs.exceptions import (
|
||||
ControlBoardReplacementActorRequired,
|
||||
InvalidMacAddress,
|
||||
@@ -67,6 +69,8 @@ app.include_router(dashboard_router)
|
||||
app.include_router(miner_detail_router)
|
||||
app.include_router(monitoring_frontend_router)
|
||||
app.include_router(monitoring_router)
|
||||
app.include_router(system_health_frontend_router)
|
||||
app.include_router(system_health_router)
|
||||
install_identity_middleware(app)
|
||||
|
||||
|
||||
@@ -34663,744 +34667,6 @@ async def minerops_inject_repair_history_modal(request, call_next):
|
||||
|
||||
|
||||
|
||||
# MINEROPS_SYSTEM_HEALTH_V1
|
||||
from fastapi.responses import HTMLResponse as MinerOpsSystemHealthHTMLResponse
|
||||
from pathlib import Path as MinerOpsSystemPath
|
||||
|
||||
|
||||
@app.get("/api/system/health-summary")
|
||||
def minerops_system_health_summary(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
ensure_location_tables()
|
||||
ensure_maintenance_tables()
|
||||
|
||||
source_path = MinerOpsSystemPath(__file__)
|
||||
source_text = source_path.read_text(errors="ignore")
|
||||
source_lines = source_text.splitlines()
|
||||
|
||||
route_re = re.compile(r'^\s*@app\.(get|post|patch|delete|put)\("([^"]+)"')
|
||||
routes = []
|
||||
|
||||
for line_no, line in enumerate(source_lines, start=1):
|
||||
match = route_re.search(line)
|
||||
|
||||
if match:
|
||||
routes.append(
|
||||
{
|
||||
"method": match.group(1).upper(),
|
||||
"path": match.group(2),
|
||||
"line": line_no,
|
||||
}
|
||||
)
|
||||
|
||||
route_counter = Counter((r["method"], r["path"]) for r in routes)
|
||||
duplicate_routes = []
|
||||
|
||||
for (method, route_path), count in sorted(route_counter.items(), key=lambda item: (item[0][1], item[0][0])):
|
||||
if count > 1:
|
||||
duplicate_routes.append(
|
||||
{
|
||||
"method": method,
|
||||
"path": route_path,
|
||||
"count": count,
|
||||
"lines": [
|
||||
r["line"]
|
||||
for r in routes
|
||||
if r["method"] == method and r["path"] == route_path
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
middleware_lines = [
|
||||
line_no
|
||||
for line_no, line in enumerate(source_lines, start=1)
|
||||
if '@app.middleware("http")' in line
|
||||
]
|
||||
|
||||
asset_routes = [
|
||||
r
|
||||
for r in routes
|
||||
if str(r["path"]).startswith("/assets/")
|
||||
]
|
||||
|
||||
patch_markers = []
|
||||
|
||||
for line_no, line in enumerate(source_lines, start=1):
|
||||
stripped = line.strip()
|
||||
|
||||
if stripped.startswith("# MINEROPS_") or stripped.startswith("# Step "):
|
||||
patch_markers.append(
|
||||
{
|
||||
"line": line_no,
|
||||
"text": stripped,
|
||||
}
|
||||
)
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
|
||||
with conn.cursor(row_factory=psycopg.rows.dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
relname AS table_name,
|
||||
pg_total_relation_size(relid) AS total_bytes,
|
||||
pg_size_pretty(pg_total_relation_size(relid)) AS total_size
|
||||
FROM pg_catalog.pg_statio_user_tables
|
||||
ORDER BY pg_total_relation_size(relid) DESC
|
||||
LIMIT 15;
|
||||
"""
|
||||
)
|
||||
largest_tables = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
relname AS table_name,
|
||||
n_live_tup::bigint AS estimated_rows,
|
||||
n_dead_tup::bigint AS estimated_dead_rows,
|
||||
last_vacuum,
|
||||
last_autovacuum,
|
||||
last_analyze,
|
||||
last_autoanalyze
|
||||
FROM pg_stat_user_tables
|
||||
ORDER BY n_live_tup DESC
|
||||
LIMIT 20;
|
||||
"""
|
||||
)
|
||||
row_estimates = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
table_counts = {}
|
||||
|
||||
for table_name in [
|
||||
"miners",
|
||||
"location_positions",
|
||||
"miner_location_states",
|
||||
"miner_maintenance_events",
|
||||
"miner_hashboards",
|
||||
"miner_network_identities",
|
||||
"minerops_agent_command_queue",
|
||||
]:
|
||||
try:
|
||||
cur.execute(f"SELECT COUNT(*) AS row_count FROM {table_name};")
|
||||
table_counts[table_name] = cur.fetchone()["row_count"]
|
||||
except Exception as exc:
|
||||
table_counts[table_name] = f"error: {exc}"
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(status, 'unknown') AS status,
|
||||
COUNT(*) AS count
|
||||
FROM miners
|
||||
GROUP BY COALESCE(status, 'unknown')
|
||||
ORDER BY status;
|
||||
"""
|
||||
)
|
||||
miner_status_counts = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(asset_state, 'unknown') AS asset_state,
|
||||
COUNT(*) AS count
|
||||
FROM miner_location_states
|
||||
GROUP BY COALESCE(asset_state, 'unknown')
|
||||
ORDER BY asset_state;
|
||||
"""
|
||||
)
|
||||
location_state_counts = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(event_status, 'unknown') AS event_status,
|
||||
COUNT(*) AS count
|
||||
FROM miner_maintenance_events
|
||||
WHERE event_type = 'repair'
|
||||
GROUP BY COALESCE(event_status, 'unknown')
|
||||
ORDER BY event_status;
|
||||
"""
|
||||
)
|
||||
repair_status_counts = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE api3_supported IS TRUE) AS api3_supported,
|
||||
COUNT(*) FILTER (WHERE api3_supported IS FALSE) AS api3_failed,
|
||||
COUNT(*) FILTER (WHERE api3_supported IS NULL) AS api3_unknown,
|
||||
COUNT(*) FILTER (WHERE power_sn IS NOT NULL AND power_sn <> '') AS miners_with_power_sn,
|
||||
COUNT(*) FILTER (WHERE miner_sn IS NOT NULL AND miner_sn <> '') AS miners_with_miner_sn
|
||||
FROM miners;
|
||||
"""
|
||||
)
|
||||
api3_summary = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS repair_queue_count
|
||||
FROM miner_location_states
|
||||
WHERE asset_state = 'repair';
|
||||
"""
|
||||
)
|
||||
repair_queue_count = cur.fetchone()["repair_queue_count"]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS raw_status_rows,
|
||||
MIN(observed_at) AS raw_oldest,
|
||||
MAX(observed_at) AS raw_newest,
|
||||
COUNT(*) FILTER (WHERE observed_at < now() - interval '30 days') AS raw_older_than_30d
|
||||
FROM miner_status_snapshots;
|
||||
"""
|
||||
)
|
||||
raw_status_retention = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS hourly_rows,
|
||||
MIN(bucket_at) AS hourly_oldest,
|
||||
MAX(bucket_at) AS hourly_newest
|
||||
FROM miner_status_hourly_rollups;
|
||||
"""
|
||||
)
|
||||
hourly_retention = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS daily_rows,
|
||||
MIN(bucket_date) AS daily_oldest,
|
||||
MAX(bucket_date) AS daily_newest
|
||||
FROM miner_status_daily_rollups;
|
||||
"""
|
||||
)
|
||||
daily_retention = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
mode,
|
||||
raw_status_retention_days,
|
||||
raw_status_rows_eligible_delete,
|
||||
raw_status_rows_deleted,
|
||||
hourly_rollup_rows_after,
|
||||
daily_rollup_rows_after,
|
||||
notes,
|
||||
started_at,
|
||||
finished_at
|
||||
FROM minerops_retention_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1;
|
||||
"""
|
||||
)
|
||||
latest_retention_run = cur.fetchone()
|
||||
latest_retention_run = dict(latest_retention_run) if latest_retention_run else None
|
||||
|
||||
return api_response(
|
||||
data={
|
||||
"app": {
|
||||
"status": "ok",
|
||||
"source_lines": len(source_lines),
|
||||
"source_size_bytes": source_path.stat().st_size,
|
||||
"route_count": len(routes),
|
||||
"duplicate_route_count": len(duplicate_routes),
|
||||
"middleware_count": len(middleware_lines),
|
||||
"asset_route_count": len(asset_routes),
|
||||
},
|
||||
"routes": {
|
||||
"duplicates": duplicate_routes,
|
||||
"middleware_lines": middleware_lines,
|
||||
"asset_routes": asset_routes,
|
||||
"recent_patch_markers": patch_markers[-40:],
|
||||
},
|
||||
"database": {
|
||||
"largest_tables": largest_tables,
|
||||
"row_estimates": row_estimates,
|
||||
"table_counts": table_counts,
|
||||
"miner_status_counts": miner_status_counts,
|
||||
"location_state_counts": location_state_counts,
|
||||
"repair_status_counts": repair_status_counts,
|
||||
"api3_summary": api3_summary,
|
||||
"repair_queue_count": repair_queue_count,
|
||||
"retention": {
|
||||
"raw_status": raw_status_retention,
|
||||
"hourly_rollups": hourly_retention,
|
||||
"daily_rollups": daily_retention,
|
||||
"latest_run": latest_retention_run,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
finally:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
MINEROPS_SYSTEM_HEALTH_HTML = r"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>MinerOps System Health</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
body {
|
||||
font-family: Segoe UI, Arial, sans-serif;
|
||||
background: #f4f7fb;
|
||||
color: #102a43;
|
||||
margin: 0;
|
||||
padding: 22px;
|
||||
}
|
||||
|
||||
.nav {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.nav a {
|
||||
border: 1px solid #bcccdc;
|
||||
border-radius: 999px;
|
||||
padding: 7px 11px;
|
||||
background: #ffffff;
|
||||
color: #0b63ce;
|
||||
font-weight: 700;
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.nav a.active {
|
||||
background: #0f766e;
|
||||
border-color: #0f766e;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0 0 4px 0;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.muted {
|
||||
color: #52606d;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.card {
|
||||
background: #ffffff;
|
||||
border: 1px solid #d9e2ec;
|
||||
border-radius: 12px;
|
||||
padding: 14px;
|
||||
margin-bottom: 14px;
|
||||
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08);
|
||||
}
|
||||
|
||||
.metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(145px, 1fr));
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.metric {
|
||||
border: 1px solid #d9e2ec;
|
||||
border-radius: 10px;
|
||||
background: #f8fafc;
|
||||
padding: 11px;
|
||||
}
|
||||
|
||||
.metric .value {
|
||||
font-size: 24px;
|
||||
font-weight: 800;
|
||||
margin-top: 2px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
th, td {
|
||||
border-bottom: 1px solid #d9e2ec;
|
||||
padding: 7px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
background: #f1f5f9;
|
||||
}
|
||||
|
||||
.table-wrap {
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
button {
|
||||
border: 1px solid #829ab1;
|
||||
border-radius: 8px;
|
||||
background: #f8fafc;
|
||||
padding: 8px 10px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button.primary {
|
||||
background: #0f766e;
|
||||
border-color: #0f766e;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
.status {
|
||||
margin: 12px 0;
|
||||
padding: 10px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.ok {
|
||||
background: #f0fdf4;
|
||||
border: 1px solid #bbf7d0;
|
||||
color: #166534;
|
||||
}
|
||||
|
||||
.bad {
|
||||
background: #fef2f2;
|
||||
border: 1px solid #fecaca;
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.grid-two {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
code {
|
||||
background: #e2e8f0;
|
||||
padding: 1px 4px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="nav">
|
||||
<a href="/dashboard">Dashboard</a>
|
||||
<a href="/repairs">Repairs</a>
|
||||
<a href="/locations">Locations</a>
|
||||
<a class="active" href="/system-health">System Health</a>
|
||||
</div>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;gap:12px;align-items:start;flex-wrap:wrap;margin-bottom:14px;">
|
||||
<div>
|
||||
<h1>System Health</h1>
|
||||
<div class="muted">Quick operational view of MinerOps app size, routes, database tables, repairs, and location state.</div>
|
||||
</div>
|
||||
<button id="refreshBtn" type="button" class="primary">Refresh</button>
|
||||
</div>
|
||||
|
||||
<div id="statusBox" class="status ok">Loading system health...</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>App Health</h2>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="muted">Status</div><div id="appStatus" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">main.py lines</div><div id="sourceLines" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Routes</div><div id="routeCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Duplicate routes</div><div id="duplicateRoutes" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Middleware</div><div id="middlewareCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Assets</div><div id="assetRouteCount" class="value">-</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Fleet / Repair State</h2>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="muted">Miners</div><div id="minerCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">In Repair</div><div id="repairCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Shelved</div><div id="shelvedCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Waiting Placement</div><div id="waitingCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">API3 Supported</div><div id="api3Supported" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">API3 Failed</div><div id="api3Failed" class="value">-</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Status Retention</h2>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="muted">Raw Status Rows</div><div id="rawStatusRows" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Older Than 30 Days</div><div id="rawOlder30" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Hourly Rollups</div><div id="hourlyRollups" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Daily Rollups</div><div id="dailyRollups" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Last Retention Mode</div><div id="lastRetentionMode" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Last Deleted Rows</div><div id="lastDeletedRows" class="value">-</div></div>
|
||||
</div>
|
||||
<div id="retentionDetails" class="muted" style="margin-top:10px;">Loading retention details...</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-two">
|
||||
<div class="card">
|
||||
<h2>Largest Tables</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Table</th><th>Size</th></tr></thead>
|
||||
<tbody id="largestTablesBody"><tr><td colspan="2">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Repair Status Counts</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Status</th><th>Count</th></tr></thead>
|
||||
<tbody id="repairStatusBody"><tr><td colspan="2">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-two">
|
||||
<div class="card">
|
||||
<h2>Duplicate Routes</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Method</th><th>Path</th><th>Count</th><th>Lines</th></tr></thead>
|
||||
<tbody id="duplicateRoutesBody"><tr><td colspan="4">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Recent Patch Markers</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Line</th><th>Marker</th></tr></thead>
|
||||
<tbody id="patchMarkersBody"><tr><td colspan="2">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
function getStoredToken() {
|
||||
const keys = [
|
||||
"minerops_token",
|
||||
"mineropsToken",
|
||||
"authToken",
|
||||
"access_token",
|
||||
"token"
|
||||
];
|
||||
|
||||
for (const storage of [window.localStorage, window.sessionStorage]) {
|
||||
if (!storage) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
const value = storage.getItem(key);
|
||||
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function authHeaders(extra) {
|
||||
const headers = Object.assign({}, extra || {});
|
||||
const token = getStoredToken();
|
||||
|
||||
if (token && !headers.Authorization) {
|
||||
headers.Authorization = "Bearer " + token;
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function apiGet(url) {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
credentials: "same-origin",
|
||||
headers: authHeaders({
|
||||
"Accept": "application/json"
|
||||
})
|
||||
});
|
||||
|
||||
const payload = await response.json().catch(function () {
|
||||
return null;
|
||||
});
|
||||
|
||||
if (!response.ok || (payload && payload.error)) {
|
||||
throw new Error((payload && payload.error) || "Request failed");
|
||||
}
|
||||
|
||||
return payload ? payload.data : null;
|
||||
}
|
||||
|
||||
function escapeHtml(value) {
|
||||
return String(value == null ? "" : value)
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/\"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function setText(id, value) {
|
||||
const el = document.getElementById(id);
|
||||
|
||||
if (el) {
|
||||
el.textContent = value == null ? "-" : value;
|
||||
}
|
||||
}
|
||||
|
||||
function tableRows(rows, columns, emptyMessage) {
|
||||
if (!rows || !rows.length) {
|
||||
return '<tr><td colspan="' + columns.length + '">' + escapeHtml(emptyMessage || "No rows") + '</td></tr>';
|
||||
}
|
||||
|
||||
return rows.map(function (row) {
|
||||
return "<tr>" + columns.map(function (col) {
|
||||
let value = row[col];
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
value = value.join(", ");
|
||||
}
|
||||
|
||||
return "<td>" + escapeHtml(value) + "</td>";
|
||||
}).join("") + "</tr>";
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function countByKey(rows, keyName, valueName) {
|
||||
const result = {};
|
||||
|
||||
for (const row of rows || []) {
|
||||
result[row[keyName]] = row[valueName];
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadHealth() {
|
||||
const statusBox = document.getElementById("statusBox");
|
||||
|
||||
try {
|
||||
statusBox.className = "status ok";
|
||||
statusBox.textContent = "Loading system health...";
|
||||
|
||||
const data = await apiGet("/api/system/health-summary");
|
||||
const app = data.app || {};
|
||||
const db = data.database || {};
|
||||
const routes = data.routes || {};
|
||||
|
||||
setText("appStatus", app.status || "-");
|
||||
setText("sourceLines", Number(app.source_lines || 0).toLocaleString());
|
||||
setText("routeCount", app.route_count || 0);
|
||||
setText("duplicateRoutes", app.duplicate_route_count || 0);
|
||||
setText("middlewareCount", app.middleware_count || 0);
|
||||
setText("assetRouteCount", app.asset_route_count || 0);
|
||||
|
||||
const locationCounts = countByKey(db.location_state_counts, "asset_state", "count");
|
||||
const tableCounts = db.table_counts || {};
|
||||
const api3 = db.api3_summary || {};
|
||||
|
||||
const retention = db.retention || {};
|
||||
const rawStatus = retention.raw_status || {};
|
||||
const hourly = retention.hourly_rollups || {};
|
||||
const daily = retention.daily_rollups || {};
|
||||
const latestRetention = retention.latest_run || {};
|
||||
|
||||
setText("minerCount", tableCounts.miners || 0);
|
||||
setText("repairCount", db.repair_queue_count || 0);
|
||||
setText("shelvedCount", locationCounts.shelved || 0);
|
||||
setText("waitingCount", locationCounts.waiting_placement || 0);
|
||||
setText("api3Supported", api3.api3_supported || 0);
|
||||
setText("api3Failed", api3.api3_failed || 0);
|
||||
|
||||
setText("rawStatusRows", Number(rawStatus.raw_status_rows || 0).toLocaleString());
|
||||
setText("rawOlder30", Number(rawStatus.raw_older_than_30d || 0).toLocaleString());
|
||||
setText("hourlyRollups", Number(hourly.hourly_rows || 0).toLocaleString());
|
||||
setText("dailyRollups", Number(daily.daily_rows || 0).toLocaleString());
|
||||
setText("lastRetentionMode", latestRetention.mode || "-");
|
||||
setText("lastDeletedRows", Number(latestRetention.raw_status_rows_deleted || 0).toLocaleString());
|
||||
|
||||
const retentionDetails = document.getElementById("retentionDetails");
|
||||
if (retentionDetails) {
|
||||
retentionDetails.innerHTML =
|
||||
"Raw range: <code>" + escapeHtml(rawStatus.raw_oldest || "-") + "</code> to <code>" + escapeHtml(rawStatus.raw_newest || "-") + "</code><br>" +
|
||||
"Hourly range: <code>" + escapeHtml(hourly.hourly_oldest || "-") + "</code> to <code>" + escapeHtml(hourly.hourly_newest || "-") + "</code><br>" +
|
||||
"Daily range: <code>" + escapeHtml(daily.daily_oldest || "-") + "</code> to <code>" + escapeHtml(daily.daily_newest || "-") + "</code><br>" +
|
||||
"Latest run: <code>" + escapeHtml(latestRetention.started_at || "-") + "</code> | " + escapeHtml(latestRetention.notes || "-");
|
||||
}
|
||||
|
||||
document.getElementById("largestTablesBody").innerHTML = tableRows(
|
||||
db.largest_tables || [],
|
||||
["table_name", "total_size"],
|
||||
"No table size data"
|
||||
);
|
||||
|
||||
document.getElementById("repairStatusBody").innerHTML = tableRows(
|
||||
db.repair_status_counts || [],
|
||||
["event_status", "count"],
|
||||
"No repair events"
|
||||
);
|
||||
|
||||
document.getElementById("duplicateRoutesBody").innerHTML = tableRows(
|
||||
routes.duplicates || [],
|
||||
["method", "path", "count", "lines"],
|
||||
"No duplicate routes"
|
||||
);
|
||||
|
||||
document.getElementById("patchMarkersBody").innerHTML = tableRows(
|
||||
routes.recent_patch_markers || [],
|
||||
["line", "text"],
|
||||
"No patch markers"
|
||||
);
|
||||
|
||||
statusBox.className = "status ok";
|
||||
statusBox.textContent = "System health loaded.";
|
||||
} catch (error) {
|
||||
statusBox.className = "status bad";
|
||||
statusBox.textContent = "System health failed to load: " + (error.message || error);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById("refreshBtn").addEventListener("click", loadHealth);
|
||||
loadHealth();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@app.get("/system-health", response_class=MinerOpsSystemHealthHTMLResponse)
|
||||
def minerops_system_health_page():
|
||||
return MINEROPS_SYSTEM_HEALTH_HTML
|
||||
|
||||
|
||||
# MINEROPS_SYSTEM_HEALTH_RETENTION_V1
|
||||
|
||||
# MINEROPS_REPAIR_READY_TO_SHELF_STATUS_V1
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""System Health diagnostics domain."""
|
||||
@@ -0,0 +1,29 @@
|
||||
"""External service adapters for System Health."""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import redis
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
RedisFactory = Callable[[], redis.Redis]
|
||||
|
||||
|
||||
def default_redis_factory() -> redis.Redis:
|
||||
return redis.Redis.from_url(
|
||||
settings.redis_url,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2,
|
||||
)
|
||||
|
||||
|
||||
class RedisHealthAdapter:
|
||||
def __init__(self, factory: RedisFactory | None = None):
|
||||
self._factory = factory or default_redis_factory
|
||||
|
||||
def ping(self) -> bool:
|
||||
try:
|
||||
return bool(self._factory().ping())
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Startup-cached, non-secret application metadata."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
def _clean_build_value(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
cleaned = value.strip()
|
||||
return cleaned[:128] if cleaned else None
|
||||
|
||||
|
||||
def _expected_migration_head() -> str | None:
|
||||
try:
|
||||
backend_root = Path(__file__).resolve().parents[2]
|
||||
config = Config(str(backend_root / "alembic.ini"))
|
||||
config.set_main_option("script_location", str(backend_root / "alembic"))
|
||||
return ScriptDirectory.from_config(config).get_current_head()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApplicationMetadata:
|
||||
environment: str
|
||||
build_commit: str | None
|
||||
build_version: str | None
|
||||
expected_migration_head: str | None
|
||||
|
||||
|
||||
APPLICATION_METADATA = ApplicationMetadata(
|
||||
environment=settings.app_env,
|
||||
build_commit=_clean_build_value(settings.minerops_build_commit),
|
||||
build_version=_clean_build_value(settings.minerops_build_version),
|
||||
expected_migration_head=_expected_migration_head(),
|
||||
)
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Dependency construction for System Health."""
|
||||
|
||||
from app.system_health.adapters import RedisHealthAdapter
|
||||
from app.system_health.repository import SystemHealthRepository
|
||||
from app.system_health.service import SystemHealthService
|
||||
|
||||
|
||||
def get_system_health_service() -> SystemHealthService:
|
||||
return SystemHealthService(SystemHealthRepository(), RedisHealthAdapter())
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Authenticated frontend routes for System Health."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import HTMLResponse, Response
|
||||
|
||||
from app.frontend.views import render_app_page
|
||||
from app.identity import require_permission, user_has_permission
|
||||
|
||||
|
||||
router = APIRouter(tags=["frontend"])
|
||||
STATIC_ROOT = Path(__file__).parents[1] / "frontend" / "static"
|
||||
SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))]
|
||||
|
||||
|
||||
@router.get("/assets/minerops-system-health.css", include_in_schema=False)
|
||||
def system_health_styles() -> Response:
|
||||
return Response((STATIC_ROOT / "system-health.css").read_text(encoding="utf-8"),
|
||||
media_type="text/css", headers={"Cache-Control": "no-store"})
|
||||
|
||||
|
||||
@router.get("/assets/minerops-system-health.js", include_in_schema=False)
|
||||
def system_health_script() -> Response:
|
||||
return Response((STATIC_ROOT / "system-health.js").read_text(encoding="utf-8"),
|
||||
media_type="text/javascript", headers={"Cache-Control": "no-store"})
|
||||
|
||||
|
||||
@router.get("/system-health", response_class=HTMLResponse, include_in_schema=False)
|
||||
def system_health_page(current_user: SystemHealthUser) -> str:
|
||||
return render_app_page(
|
||||
"system_health", current_user,
|
||||
lambda permission: user_has_permission(current_user, permission),
|
||||
)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Typed contracts for the System Health overview."""
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
Availability = Literal["available", "unavailable"]
|
||||
MigrationState = Literal["current", "mismatch", "unavailable"]
|
||||
|
||||
|
||||
class ApplicationDiagnostics(BaseModel):
|
||||
status: Literal["available"] = "available"
|
||||
environment: str
|
||||
build_commit: str | None = None
|
||||
build_version: str | None = None
|
||||
server_time: datetime
|
||||
expected_migration_head: str | None = None
|
||||
|
||||
|
||||
class MigrationDiagnostics(BaseModel):
|
||||
state: MigrationState
|
||||
current_revision: str | None = None
|
||||
expected_revision: str | None = None
|
||||
|
||||
|
||||
class TableCount(BaseModel):
|
||||
name: str
|
||||
row_count: int
|
||||
|
||||
|
||||
class TableSize(BaseModel):
|
||||
name: str
|
||||
total_bytes: int
|
||||
|
||||
|
||||
class TelemetryDiagnostics(BaseModel):
|
||||
snapshot_count: int
|
||||
oldest_observed_at: datetime | None = None
|
||||
latest_observed_at: datetime | None = None
|
||||
|
||||
|
||||
class FleetIntegrityDiagnostics(BaseModel):
|
||||
total_miners: int
|
||||
miners_without_asset_id: int
|
||||
unmapped_miners: int
|
||||
miners_in_multiple_positions: int
|
||||
duplicate_current_mac_groups: int
|
||||
|
||||
|
||||
class LocationIntegrityDiagnostics(BaseModel):
|
||||
sites: int
|
||||
rooms: int
|
||||
positions: int
|
||||
occupied_positions: int
|
||||
empty_positions: int
|
||||
unscoped_legacy_positions: int
|
||||
expected_mac_mismatches: int
|
||||
|
||||
|
||||
class RepairIntegrityDiagnostics(BaseModel):
|
||||
active_modular_repairs: int
|
||||
duplicate_active_repair_groups: int
|
||||
legacy_repair_only_miners: int
|
||||
lifecycle_repair_without_active_repair: int
|
||||
active_repair_lifecycle_mismatches: int
|
||||
waiting_placement_records: int
|
||||
repair_events: int
|
||||
repair_parts: int
|
||||
|
||||
|
||||
class RollupDiagnostics(BaseModel):
|
||||
table_present: bool
|
||||
row_count: int | None = None
|
||||
latest_bucket: datetime | date | None = None
|
||||
population: Literal["manual"] = "manual"
|
||||
|
||||
|
||||
class RetentionDiagnostics(BaseModel):
|
||||
raw_retention_enabled: Literal[False] = False
|
||||
raw_retention_mode: Literal["not_enabled"] = "not_enabled"
|
||||
hourly: RollupDiagnostics
|
||||
daily: RollupDiagnostics
|
||||
latest_recorded_run_at: datetime | None = None
|
||||
latest_recorded_run_mode: str | None = None
|
||||
completeness: Literal["not_guaranteed"] = "not_guaranteed"
|
||||
|
||||
|
||||
class PostgreSQLDiagnostics(BaseModel):
|
||||
status: Availability
|
||||
migration: MigrationDiagnostics
|
||||
table_counts: list[TableCount] = Field(default_factory=list)
|
||||
largest_tables: list[TableSize] = Field(default_factory=list)
|
||||
telemetry: TelemetryDiagnostics | None = None
|
||||
fleet: FleetIntegrityDiagnostics | None = None
|
||||
locations: LocationIntegrityDiagnostics | None = None
|
||||
repairs: RepairIntegrityDiagnostics | None = None
|
||||
retention: RetentionDiagnostics | None = None
|
||||
|
||||
|
||||
class RedisDiagnostics(BaseModel):
|
||||
status: Availability
|
||||
|
||||
|
||||
class SystemHealthOverviewData(BaseModel):
|
||||
application: ApplicationDiagnostics
|
||||
postgresql: PostgreSQLDiagnostics
|
||||
redis: RedisDiagnostics
|
||||
|
||||
|
||||
class SystemHealthOverviewMeta(BaseModel):
|
||||
generated_at: datetime
|
||||
partial: bool
|
||||
unavailable_sections: list[str] = Field(default_factory=list)
|
||||
frontend_request_count: Literal[1] = 1
|
||||
postgresql_statement_count: Literal[3] = 3
|
||||
redis_call_count: Literal[1] = 1
|
||||
|
||||
|
||||
class SystemHealthOverviewResponse(BaseModel):
|
||||
data: SystemHealthOverviewData
|
||||
meta: SystemHealthOverviewMeta
|
||||
error: None = None
|
||||
@@ -0,0 +1,179 @@
|
||||
"""PostgreSQL diagnostics for System Health."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
ConnectionFactory = Callable[[], psycopg.Connection]
|
||||
|
||||
|
||||
def default_connection_factory() -> psycopg.Connection:
|
||||
return psycopg.connect(
|
||||
host=settings.postgres_host,
|
||||
port=settings.postgres_port,
|
||||
dbname=settings.postgres_db,
|
||||
user=settings.postgres_user,
|
||||
password=settings.postgres_password,
|
||||
connect_timeout=3,
|
||||
)
|
||||
|
||||
|
||||
MIGRATION_SQL = """
|
||||
SELECT clock_timestamp() AS database_time,
|
||||
(SELECT version_num FROM alembic_version LIMIT 1) AS current_revision
|
||||
"""
|
||||
|
||||
OPERATIONS_SQL = """
|
||||
WITH telemetry AS (
|
||||
SELECT count(*)::bigint AS snapshot_count,
|
||||
min(observed_at) AS oldest_observed_at,
|
||||
max(observed_at) AS latest_observed_at
|
||||
FROM miner_status_snapshots
|
||||
),
|
||||
hourly AS (
|
||||
SELECT count(*)::bigint AS row_count, max(bucket_at) AS latest_bucket
|
||||
FROM miner_status_hourly_rollups
|
||||
),
|
||||
daily AS (
|
||||
SELECT count(*)::bigint AS row_count, max(bucket_date) AS latest_bucket
|
||||
FROM miner_status_daily_rollups
|
||||
),
|
||||
latest_retention AS (
|
||||
SELECT mode, started_at FROM minerops_retention_runs
|
||||
ORDER BY started_at DESC LIMIT 1
|
||||
),
|
||||
largest AS (
|
||||
SELECT coalesce(jsonb_agg(to_jsonb(sized) ORDER BY sized.total_bytes DESC), '[]'::jsonb) AS rows
|
||||
FROM (
|
||||
SELECT relname AS name, pg_total_relation_size(relid)::bigint AS total_bytes
|
||||
FROM pg_catalog.pg_statio_user_tables
|
||||
WHERE relname = ANY(%s)
|
||||
ORDER BY pg_total_relation_size(relid) DESC, relname
|
||||
LIMIT 8
|
||||
) AS sized
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM miners)::bigint AS miners,
|
||||
(SELECT count(*) FROM sites)::bigint AS sites,
|
||||
(SELECT count(*) FROM rooms)::bigint AS rooms,
|
||||
(SELECT count(*) FROM location_positions)::bigint AS location_positions,
|
||||
(SELECT count(*) FROM repairs)::bigint AS repairs,
|
||||
(SELECT count(*) FROM repair_events)::bigint AS repair_events,
|
||||
(SELECT count(*) FROM repair_parts)::bigint AS repair_parts,
|
||||
telemetry.snapshot_count, telemetry.oldest_observed_at,
|
||||
telemetry.latest_observed_at,
|
||||
to_regclass('public.miner_status_hourly_rollups') IS NOT NULL AS hourly_present,
|
||||
hourly.row_count AS hourly_rows, hourly.latest_bucket AS hourly_latest,
|
||||
to_regclass('public.miner_status_daily_rollups') IS NOT NULL AS daily_present,
|
||||
daily.row_count AS daily_rows, daily.latest_bucket AS daily_latest,
|
||||
latest_retention.mode AS retention_mode,
|
||||
latest_retention.started_at AS retention_started_at,
|
||||
largest.rows AS largest_tables
|
||||
FROM telemetry CROSS JOIN hourly CROSS JOIN daily CROSS JOIN largest
|
||||
LEFT JOIN latest_retention ON true
|
||||
"""
|
||||
|
||||
INTEGRITY_SQL = """
|
||||
WITH position_counts AS (
|
||||
SELECT miner_id, count(*) AS position_count
|
||||
FROM location_positions WHERE miner_id IS NOT NULL GROUP BY miner_id
|
||||
),
|
||||
active_repairs AS (
|
||||
SELECT id, miner_id FROM repairs
|
||||
WHERE status IN ('OPEN', 'IN_PROGRESS', 'WAITING_FOR_PARTS', 'READY_FOR_TEST')
|
||||
),
|
||||
active_repair_groups AS (
|
||||
SELECT miner_id FROM active_repairs GROUP BY miner_id HAVING count(*) > 1
|
||||
),
|
||||
normalized_macs AS (
|
||||
SELECT lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g')) AS mac
|
||||
FROM miners WHERE current_mac IS NOT NULL AND trim(current_mac) <> ''
|
||||
GROUP BY lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g'))
|
||||
HAVING count(*) > 1
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM miners WHERE asset_tag IS NULL OR trim(asset_tag) = '')::bigint
|
||||
AS miners_without_asset_id,
|
||||
(SELECT count(*) FROM miners m WHERE NOT EXISTS (
|
||||
SELECT 1 FROM location_positions p WHERE p.miner_id = m.id
|
||||
))::bigint AS unmapped_miners,
|
||||
(SELECT count(*) FROM position_counts WHERE position_count > 1)::bigint
|
||||
AS miners_in_multiple_positions,
|
||||
(SELECT count(*) FROM normalized_macs)::bigint AS duplicate_current_mac_groups,
|
||||
(SELECT count(*) FROM location_positions WHERE room_id IS NULL)::bigint
|
||||
AS unscoped_legacy_positions,
|
||||
(SELECT count(*) FROM location_positions p JOIN miners m ON m.id = p.miner_id
|
||||
WHERE p.expected_mac IS NOT NULL AND trim(p.expected_mac) <> ''
|
||||
AND m.current_mac IS NOT NULL AND trim(m.current_mac) <> ''
|
||||
AND lower(regexp_replace(trim(p.expected_mac), '[^0-9A-Fa-f]', '', 'g'))
|
||||
<> lower(regexp_replace(trim(m.current_mac), '[^0-9A-Fa-f]', '', 'g'))
|
||||
)::bigint AS expected_mac_mismatches,
|
||||
(SELECT count(*) FROM location_positions WHERE miner_id IS NOT NULL)::bigint
|
||||
AS occupied_positions,
|
||||
(SELECT count(*) FROM location_positions WHERE miner_id IS NULL)::bigint
|
||||
AS empty_positions,
|
||||
(SELECT count(*) FROM active_repairs)::bigint AS active_modular_repairs,
|
||||
(SELECT count(*) FROM active_repair_groups)::bigint AS duplicate_active_repair_groups,
|
||||
(SELECT count(*) FROM miner_location_states state
|
||||
WHERE state.asset_state = 'repair' AND NOT EXISTS (
|
||||
SELECT 1 FROM active_repairs repair WHERE repair.miner_id = state.miner_id
|
||||
))::bigint AS legacy_repair_only_miners,
|
||||
(SELECT count(*) FROM miners m WHERE m.lifecycle_status = 'in_repair'
|
||||
AND NOT EXISTS (SELECT 1 FROM active_repairs r WHERE r.miner_id = m.id))::bigint
|
||||
AS lifecycle_repair_without_active_repair,
|
||||
(SELECT count(*) FROM active_repairs r JOIN miners m ON m.id = r.miner_id
|
||||
WHERE m.lifecycle_status <> 'in_repair')::bigint
|
||||
AS active_repair_lifecycle_mismatches,
|
||||
(SELECT count(*) FROM miner_location_states WHERE asset_state = 'waiting_placement')::bigint
|
||||
AS waiting_placement_records
|
||||
"""
|
||||
|
||||
|
||||
RELEVANT_TABLES = [
|
||||
"miner_status_snapshots", "repair_events", "repair_parts", "repairs",
|
||||
"location_positions", "miners", "miner_status_hourly_rollups",
|
||||
"miner_status_daily_rollups",
|
||||
]
|
||||
|
||||
|
||||
class SystemHealthRepository:
|
||||
def __init__(
|
||||
self,
|
||||
connection_factory: ConnectionFactory | None = None,
|
||||
*,
|
||||
connection: psycopg.Connection | None = None,
|
||||
):
|
||||
self._connection_factory = connection_factory or default_connection_factory
|
||||
self._connection = connection
|
||||
|
||||
@contextmanager
|
||||
def _connection_scope(self) -> Iterator[psycopg.Connection]:
|
||||
if self._connection is not None:
|
||||
yield self._connection
|
||||
return
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
yield connection
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def read_overview(self) -> dict[str, Any]:
|
||||
with self._connection_scope() as connection:
|
||||
with connection.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(MIGRATION_SQL)
|
||||
migration = dict(cur.fetchone())
|
||||
cur.execute(OPERATIONS_SQL, (RELEVANT_TABLES,))
|
||||
operations = dict(cur.fetchone())
|
||||
cur.execute(INTEGRITY_SQL)
|
||||
integrity = dict(cur.fetchone())
|
||||
return {
|
||||
"migration": migration,
|
||||
"operations": operations,
|
||||
"integrity": integrity,
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
"""HTTP boundary for the System Health overview."""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Response
|
||||
|
||||
from app.identity import require_permission
|
||||
from app.system_health.dependencies import get_system_health_service
|
||||
from app.system_health.models import SystemHealthOverviewResponse
|
||||
from app.system_health.service import SystemHealthService
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/system", tags=["system-health"])
|
||||
SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))]
|
||||
SystemHealthServiceDependency = Annotated[
|
||||
SystemHealthService, Depends(get_system_health_service)
|
||||
]
|
||||
|
||||
|
||||
@router.get("/health-summary", response_model=SystemHealthOverviewResponse)
|
||||
def get_system_health_overview(
|
||||
current_user: SystemHealthUser,
|
||||
service: SystemHealthServiceDependency,
|
||||
response: Response,
|
||||
) -> SystemHealthOverviewResponse:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return service.get_overview()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Composition and failure semantics for System Health diagnostics."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.system_health.adapters import RedisHealthAdapter
|
||||
from app.system_health.application import APPLICATION_METADATA, ApplicationMetadata
|
||||
from app.system_health.models import (
|
||||
ApplicationDiagnostics, FleetIntegrityDiagnostics, LocationIntegrityDiagnostics,
|
||||
MigrationDiagnostics, PostgreSQLDiagnostics, RedisDiagnostics,
|
||||
RepairIntegrityDiagnostics, RetentionDiagnostics, RollupDiagnostics,
|
||||
SystemHealthOverviewData, SystemHealthOverviewMeta,
|
||||
SystemHealthOverviewResponse, TableCount, TableSize, TelemetryDiagnostics,
|
||||
)
|
||||
from app.system_health.repository import SystemHealthRepository
|
||||
|
||||
|
||||
class SystemHealthService:
|
||||
def __init__(self, repository: SystemHealthRepository,
|
||||
redis_adapter: RedisHealthAdapter,
|
||||
metadata: ApplicationMetadata = APPLICATION_METADATA):
|
||||
self.repository = repository
|
||||
self.redis_adapter = redis_adapter
|
||||
self.metadata = metadata
|
||||
|
||||
def get_overview(self) -> SystemHealthOverviewResponse:
|
||||
generated_at = datetime.now(timezone.utc)
|
||||
unavailable = []
|
||||
try:
|
||||
postgresql = self._postgresql_diagnostics(
|
||||
self.repository.read_overview()
|
||||
)
|
||||
except Exception:
|
||||
unavailable.append("postgresql")
|
||||
postgresql = PostgreSQLDiagnostics(
|
||||
status="unavailable",
|
||||
migration=MigrationDiagnostics(
|
||||
state="unavailable",
|
||||
expected_revision=self.metadata.expected_migration_head,
|
||||
),
|
||||
)
|
||||
redis_available = self.redis_adapter.ping()
|
||||
if not redis_available:
|
||||
unavailable.append("redis")
|
||||
return SystemHealthOverviewResponse(
|
||||
data=SystemHealthOverviewData(
|
||||
application=ApplicationDiagnostics(
|
||||
environment=self.metadata.environment,
|
||||
build_commit=self.metadata.build_commit,
|
||||
build_version=self.metadata.build_version,
|
||||
server_time=generated_at,
|
||||
expected_migration_head=self.metadata.expected_migration_head,
|
||||
),
|
||||
postgresql=postgresql,
|
||||
redis=RedisDiagnostics(
|
||||
status="available" if redis_available else "unavailable"
|
||||
),
|
||||
),
|
||||
meta=SystemHealthOverviewMeta(
|
||||
generated_at=generated_at,
|
||||
partial=bool(unavailable),
|
||||
unavailable_sections=unavailable,
|
||||
),
|
||||
)
|
||||
|
||||
def _postgresql_diagnostics(self, payload: dict) -> PostgreSQLDiagnostics:
|
||||
migration = payload["migration"]
|
||||
operations = payload["operations"]
|
||||
integrity = payload["integrity"]
|
||||
current = migration.get("current_revision")
|
||||
expected = self.metadata.expected_migration_head
|
||||
state = (
|
||||
"unavailable" if not current or not expected
|
||||
else "current" if current == expected else "mismatch"
|
||||
)
|
||||
counts = [
|
||||
TableCount(name=name, row_count=int(operations[name]))
|
||||
for name in (
|
||||
"miners", "sites", "rooms", "location_positions", "repairs",
|
||||
"repair_events", "repair_parts", "snapshot_count",
|
||||
)
|
||||
]
|
||||
return PostgreSQLDiagnostics(
|
||||
status="available",
|
||||
migration=MigrationDiagnostics(
|
||||
state=state, current_revision=current, expected_revision=expected
|
||||
),
|
||||
table_counts=counts,
|
||||
largest_tables=[
|
||||
TableSize(name=row["name"], total_bytes=int(row["total_bytes"]))
|
||||
for row in operations.get("largest_tables") or []
|
||||
],
|
||||
telemetry=TelemetryDiagnostics(
|
||||
snapshot_count=int(operations["snapshot_count"]),
|
||||
oldest_observed_at=operations.get("oldest_observed_at"),
|
||||
latest_observed_at=operations.get("latest_observed_at"),
|
||||
),
|
||||
fleet=FleetIntegrityDiagnostics(
|
||||
total_miners=int(operations["miners"]),
|
||||
miners_without_asset_id=int(integrity["miners_without_asset_id"]),
|
||||
unmapped_miners=int(integrity["unmapped_miners"]),
|
||||
miners_in_multiple_positions=int(integrity["miners_in_multiple_positions"]),
|
||||
duplicate_current_mac_groups=int(integrity["duplicate_current_mac_groups"]),
|
||||
),
|
||||
locations=LocationIntegrityDiagnostics(
|
||||
sites=int(operations["sites"]), rooms=int(operations["rooms"]),
|
||||
positions=int(operations["location_positions"]),
|
||||
occupied_positions=int(integrity["occupied_positions"]),
|
||||
empty_positions=int(integrity["empty_positions"]),
|
||||
unscoped_legacy_positions=int(integrity["unscoped_legacy_positions"]),
|
||||
expected_mac_mismatches=int(integrity["expected_mac_mismatches"]),
|
||||
),
|
||||
repairs=RepairIntegrityDiagnostics(
|
||||
active_modular_repairs=int(integrity["active_modular_repairs"]),
|
||||
duplicate_active_repair_groups=int(integrity["duplicate_active_repair_groups"]),
|
||||
legacy_repair_only_miners=int(integrity["legacy_repair_only_miners"]),
|
||||
lifecycle_repair_without_active_repair=int(integrity["lifecycle_repair_without_active_repair"]),
|
||||
active_repair_lifecycle_mismatches=int(integrity["active_repair_lifecycle_mismatches"]),
|
||||
waiting_placement_records=int(integrity["waiting_placement_records"]),
|
||||
repair_events=int(operations["repair_events"]),
|
||||
repair_parts=int(operations["repair_parts"]),
|
||||
),
|
||||
retention=RetentionDiagnostics(
|
||||
hourly=RollupDiagnostics(
|
||||
table_present=bool(operations["hourly_present"]),
|
||||
row_count=int(operations["hourly_rows"]),
|
||||
latest_bucket=operations.get("hourly_latest"),
|
||||
),
|
||||
daily=RollupDiagnostics(
|
||||
table_present=bool(operations["daily_present"]),
|
||||
row_count=int(operations["daily_rows"]),
|
||||
latest_bucket=operations.get("daily_latest"),
|
||||
),
|
||||
latest_recorded_run_at=operations.get("retention_started_at"),
|
||||
latest_recorded_run_mode=operations.get("retention_mode"),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user