feat: add system health diagnostics
This commit is contained in:
lussierj committed
1 parent
0dd26a3f24
commit
c3501b987e
17 files changed
+1335
-741
No files matched your search
+4
-738
@@ -32,6 +32,8 @@ from app.components.router import router as components_router
|
||||
from app.miner_detail.router import router as miner_detail_router
|
||||
from app.monitoring.router import router as monitoring_router
|
||||
from app.monitoring.frontend import router as monitoring_frontend_router
|
||||
from app.system_health.frontend import router as system_health_frontend_router
|
||||
from app.system_health.router import router as system_health_router
|
||||
from app.repairs.exceptions import (
|
||||
ControlBoardReplacementActorRequired,
|
||||
InvalidMacAddress,
|
||||
@@ -67,6 +69,8 @@ app.include_router(dashboard_router)
|
||||
app.include_router(miner_detail_router)
|
||||
app.include_router(monitoring_frontend_router)
|
||||
app.include_router(monitoring_router)
|
||||
app.include_router(system_health_frontend_router)
|
||||
app.include_router(system_health_router)
|
||||
install_identity_middleware(app)
|
||||
|
||||
|
||||
@@ -34663,744 +34667,6 @@ async def minerops_inject_repair_history_modal(request, call_next):
|
||||
|
||||
|
||||
|
||||
# MINEROPS_SYSTEM_HEALTH_V1
|
||||
from fastapi.responses import HTMLResponse as MinerOpsSystemHealthHTMLResponse
|
||||
from pathlib import Path as MinerOpsSystemPath
|
||||
|
||||
|
||||
@app.get("/api/system/health-summary")
|
||||
def minerops_system_health_summary(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
ensure_location_tables()
|
||||
ensure_maintenance_tables()
|
||||
|
||||
source_path = MinerOpsSystemPath(__file__)
|
||||
source_text = source_path.read_text(errors="ignore")
|
||||
source_lines = source_text.splitlines()
|
||||
|
||||
route_re = re.compile(r'^\s*@app\.(get|post|patch|delete|put)\("([^"]+)"')
|
||||
routes = []
|
||||
|
||||
for line_no, line in enumerate(source_lines, start=1):
|
||||
match = route_re.search(line)
|
||||
|
||||
if match:
|
||||
routes.append(
|
||||
{
|
||||
"method": match.group(1).upper(),
|
||||
"path": match.group(2),
|
||||
"line": line_no,
|
||||
}
|
||||
)
|
||||
|
||||
route_counter = Counter((r["method"], r["path"]) for r in routes)
|
||||
duplicate_routes = []
|
||||
|
||||
for (method, route_path), count in sorted(route_counter.items(), key=lambda item: (item[0][1], item[0][0])):
|
||||
if count > 1:
|
||||
duplicate_routes.append(
|
||||
{
|
||||
"method": method,
|
||||
"path": route_path,
|
||||
"count": count,
|
||||
"lines": [
|
||||
r["line"]
|
||||
for r in routes
|
||||
if r["method"] == method and r["path"] == route_path
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
middleware_lines = [
|
||||
line_no
|
||||
for line_no, line in enumerate(source_lines, start=1)
|
||||
if '@app.middleware("http")' in line
|
||||
]
|
||||
|
||||
asset_routes = [
|
||||
r
|
||||
for r in routes
|
||||
if str(r["path"]).startswith("/assets/")
|
||||
]
|
||||
|
||||
patch_markers = []
|
||||
|
||||
for line_no, line in enumerate(source_lines, start=1):
|
||||
stripped = line.strip()
|
||||
|
||||
if stripped.startswith("# MINEROPS_") or stripped.startswith("# Step "):
|
||||
patch_markers.append(
|
||||
{
|
||||
"line": line_no,
|
||||
"text": stripped,
|
||||
}
|
||||
)
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
|
||||
with conn.cursor(row_factory=psycopg.rows.dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
relname AS table_name,
|
||||
pg_total_relation_size(relid) AS total_bytes,
|
||||
pg_size_pretty(pg_total_relation_size(relid)) AS total_size
|
||||
FROM pg_catalog.pg_statio_user_tables
|
||||
ORDER BY pg_total_relation_size(relid) DESC
|
||||
LIMIT 15;
|
||||
"""
|
||||
)
|
||||
largest_tables = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
relname AS table_name,
|
||||
n_live_tup::bigint AS estimated_rows,
|
||||
n_dead_tup::bigint AS estimated_dead_rows,
|
||||
last_vacuum,
|
||||
last_autovacuum,
|
||||
last_analyze,
|
||||
last_autoanalyze
|
||||
FROM pg_stat_user_tables
|
||||
ORDER BY n_live_tup DESC
|
||||
LIMIT 20;
|
||||
"""
|
||||
)
|
||||
row_estimates = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
table_counts = {}
|
||||
|
||||
for table_name in [
|
||||
"miners",
|
||||
"location_positions",
|
||||
"miner_location_states",
|
||||
"miner_maintenance_events",
|
||||
"miner_hashboards",
|
||||
"miner_network_identities",
|
||||
"minerops_agent_command_queue",
|
||||
]:
|
||||
try:
|
||||
cur.execute(f"SELECT COUNT(*) AS row_count FROM {table_name};")
|
||||
table_counts[table_name] = cur.fetchone()["row_count"]
|
||||
except Exception as exc:
|
||||
table_counts[table_name] = f"error: {exc}"
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(status, 'unknown') AS status,
|
||||
COUNT(*) AS count
|
||||
FROM miners
|
||||
GROUP BY COALESCE(status, 'unknown')
|
||||
ORDER BY status;
|
||||
"""
|
||||
)
|
||||
miner_status_counts = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(asset_state, 'unknown') AS asset_state,
|
||||
COUNT(*) AS count
|
||||
FROM miner_location_states
|
||||
GROUP BY COALESCE(asset_state, 'unknown')
|
||||
ORDER BY asset_state;
|
||||
"""
|
||||
)
|
||||
location_state_counts = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(event_status, 'unknown') AS event_status,
|
||||
COUNT(*) AS count
|
||||
FROM miner_maintenance_events
|
||||
WHERE event_type = 'repair'
|
||||
GROUP BY COALESCE(event_status, 'unknown')
|
||||
ORDER BY event_status;
|
||||
"""
|
||||
)
|
||||
repair_status_counts = [dict(row) for row in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE api3_supported IS TRUE) AS api3_supported,
|
||||
COUNT(*) FILTER (WHERE api3_supported IS FALSE) AS api3_failed,
|
||||
COUNT(*) FILTER (WHERE api3_supported IS NULL) AS api3_unknown,
|
||||
COUNT(*) FILTER (WHERE power_sn IS NOT NULL AND power_sn <> '') AS miners_with_power_sn,
|
||||
COUNT(*) FILTER (WHERE miner_sn IS NOT NULL AND miner_sn <> '') AS miners_with_miner_sn
|
||||
FROM miners;
|
||||
"""
|
||||
)
|
||||
api3_summary = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS repair_queue_count
|
||||
FROM miner_location_states
|
||||
WHERE asset_state = 'repair';
|
||||
"""
|
||||
)
|
||||
repair_queue_count = cur.fetchone()["repair_queue_count"]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS raw_status_rows,
|
||||
MIN(observed_at) AS raw_oldest,
|
||||
MAX(observed_at) AS raw_newest,
|
||||
COUNT(*) FILTER (WHERE observed_at < now() - interval '30 days') AS raw_older_than_30d
|
||||
FROM miner_status_snapshots;
|
||||
"""
|
||||
)
|
||||
raw_status_retention = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS hourly_rows,
|
||||
MIN(bucket_at) AS hourly_oldest,
|
||||
MAX(bucket_at) AS hourly_newest
|
||||
FROM miner_status_hourly_rollups;
|
||||
"""
|
||||
)
|
||||
hourly_retention = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS daily_rows,
|
||||
MIN(bucket_date) AS daily_oldest,
|
||||
MAX(bucket_date) AS daily_newest
|
||||
FROM miner_status_daily_rollups;
|
||||
"""
|
||||
)
|
||||
daily_retention = dict(cur.fetchone())
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
mode,
|
||||
raw_status_retention_days,
|
||||
raw_status_rows_eligible_delete,
|
||||
raw_status_rows_deleted,
|
||||
hourly_rollup_rows_after,
|
||||
daily_rollup_rows_after,
|
||||
notes,
|
||||
started_at,
|
||||
finished_at
|
||||
FROM minerops_retention_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1;
|
||||
"""
|
||||
)
|
||||
latest_retention_run = cur.fetchone()
|
||||
latest_retention_run = dict(latest_retention_run) if latest_retention_run else None
|
||||
|
||||
return api_response(
|
||||
data={
|
||||
"app": {
|
||||
"status": "ok",
|
||||
"source_lines": len(source_lines),
|
||||
"source_size_bytes": source_path.stat().st_size,
|
||||
"route_count": len(routes),
|
||||
"duplicate_route_count": len(duplicate_routes),
|
||||
"middleware_count": len(middleware_lines),
|
||||
"asset_route_count": len(asset_routes),
|
||||
},
|
||||
"routes": {
|
||||
"duplicates": duplicate_routes,
|
||||
"middleware_lines": middleware_lines,
|
||||
"asset_routes": asset_routes,
|
||||
"recent_patch_markers": patch_markers[-40:],
|
||||
},
|
||||
"database": {
|
||||
"largest_tables": largest_tables,
|
||||
"row_estimates": row_estimates,
|
||||
"table_counts": table_counts,
|
||||
"miner_status_counts": miner_status_counts,
|
||||
"location_state_counts": location_state_counts,
|
||||
"repair_status_counts": repair_status_counts,
|
||||
"api3_summary": api3_summary,
|
||||
"repair_queue_count": repair_queue_count,
|
||||
"retention": {
|
||||
"raw_status": raw_status_retention,
|
||||
"hourly_rollups": hourly_retention,
|
||||
"daily_rollups": daily_retention,
|
||||
"latest_run": latest_retention_run,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
finally:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
MINEROPS_SYSTEM_HEALTH_HTML = r"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>MinerOps System Health</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
body {
|
||||
font-family: Segoe UI, Arial, sans-serif;
|
||||
background: #f4f7fb;
|
||||
color: #102a43;
|
||||
margin: 0;
|
||||
padding: 22px;
|
||||
}
|
||||
|
||||
.nav {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.nav a {
|
||||
border: 1px solid #bcccdc;
|
||||
border-radius: 999px;
|
||||
padding: 7px 11px;
|
||||
background: #ffffff;
|
||||
color: #0b63ce;
|
||||
font-weight: 700;
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.nav a.active {
|
||||
background: #0f766e;
|
||||
border-color: #0f766e;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0 0 4px 0;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.muted {
|
||||
color: #52606d;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.card {
|
||||
background: #ffffff;
|
||||
border: 1px solid #d9e2ec;
|
||||
border-radius: 12px;
|
||||
padding: 14px;
|
||||
margin-bottom: 14px;
|
||||
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08);
|
||||
}
|
||||
|
||||
.metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(145px, 1fr));
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.metric {
|
||||
border: 1px solid #d9e2ec;
|
||||
border-radius: 10px;
|
||||
background: #f8fafc;
|
||||
padding: 11px;
|
||||
}
|
||||
|
||||
.metric .value {
|
||||
font-size: 24px;
|
||||
font-weight: 800;
|
||||
margin-top: 2px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
th, td {
|
||||
border-bottom: 1px solid #d9e2ec;
|
||||
padding: 7px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
background: #f1f5f9;
|
||||
}
|
||||
|
||||
.table-wrap {
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
button {
|
||||
border: 1px solid #829ab1;
|
||||
border-radius: 8px;
|
||||
background: #f8fafc;
|
||||
padding: 8px 10px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button.primary {
|
||||
background: #0f766e;
|
||||
border-color: #0f766e;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
.status {
|
||||
margin: 12px 0;
|
||||
padding: 10px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.ok {
|
||||
background: #f0fdf4;
|
||||
border: 1px solid #bbf7d0;
|
||||
color: #166534;
|
||||
}
|
||||
|
||||
.bad {
|
||||
background: #fef2f2;
|
||||
border: 1px solid #fecaca;
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.grid-two {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
code {
|
||||
background: #e2e8f0;
|
||||
padding: 1px 4px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="nav">
|
||||
<a href="/dashboard">Dashboard</a>
|
||||
<a href="/repairs">Repairs</a>
|
||||
<a href="/locations">Locations</a>
|
||||
<a class="active" href="/system-health">System Health</a>
|
||||
</div>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;gap:12px;align-items:start;flex-wrap:wrap;margin-bottom:14px;">
|
||||
<div>
|
||||
<h1>System Health</h1>
|
||||
<div class="muted">Quick operational view of MinerOps app size, routes, database tables, repairs, and location state.</div>
|
||||
</div>
|
||||
<button id="refreshBtn" type="button" class="primary">Refresh</button>
|
||||
</div>
|
||||
|
||||
<div id="statusBox" class="status ok">Loading system health...</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>App Health</h2>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="muted">Status</div><div id="appStatus" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">main.py lines</div><div id="sourceLines" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Routes</div><div id="routeCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Duplicate routes</div><div id="duplicateRoutes" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Middleware</div><div id="middlewareCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Assets</div><div id="assetRouteCount" class="value">-</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Fleet / Repair State</h2>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="muted">Miners</div><div id="minerCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">In Repair</div><div id="repairCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Shelved</div><div id="shelvedCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Waiting Placement</div><div id="waitingCount" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">API3 Supported</div><div id="api3Supported" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">API3 Failed</div><div id="api3Failed" class="value">-</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Status Retention</h2>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="muted">Raw Status Rows</div><div id="rawStatusRows" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Older Than 30 Days</div><div id="rawOlder30" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Hourly Rollups</div><div id="hourlyRollups" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Daily Rollups</div><div id="dailyRollups" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Last Retention Mode</div><div id="lastRetentionMode" class="value">-</div></div>
|
||||
<div class="metric"><div class="muted">Last Deleted Rows</div><div id="lastDeletedRows" class="value">-</div></div>
|
||||
</div>
|
||||
<div id="retentionDetails" class="muted" style="margin-top:10px;">Loading retention details...</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-two">
|
||||
<div class="card">
|
||||
<h2>Largest Tables</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Table</th><th>Size</th></tr></thead>
|
||||
<tbody id="largestTablesBody"><tr><td colspan="2">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Repair Status Counts</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Status</th><th>Count</th></tr></thead>
|
||||
<tbody id="repairStatusBody"><tr><td colspan="2">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-two">
|
||||
<div class="card">
|
||||
<h2>Duplicate Routes</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Method</th><th>Path</th><th>Count</th><th>Lines</th></tr></thead>
|
||||
<tbody id="duplicateRoutesBody"><tr><td colspan="4">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Recent Patch Markers</h2>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Line</th><th>Marker</th></tr></thead>
|
||||
<tbody id="patchMarkersBody"><tr><td colspan="2">Loading...</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
function getStoredToken() {
|
||||
const keys = [
|
||||
"minerops_token",
|
||||
"mineropsToken",
|
||||
"authToken",
|
||||
"access_token",
|
||||
"token"
|
||||
];
|
||||
|
||||
for (const storage of [window.localStorage, window.sessionStorage]) {
|
||||
if (!storage) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
const value = storage.getItem(key);
|
||||
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function authHeaders(extra) {
|
||||
const headers = Object.assign({}, extra || {});
|
||||
const token = getStoredToken();
|
||||
|
||||
if (token && !headers.Authorization) {
|
||||
headers.Authorization = "Bearer " + token;
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function apiGet(url) {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
credentials: "same-origin",
|
||||
headers: authHeaders({
|
||||
"Accept": "application/json"
|
||||
})
|
||||
});
|
||||
|
||||
const payload = await response.json().catch(function () {
|
||||
return null;
|
||||
});
|
||||
|
||||
if (!response.ok || (payload && payload.error)) {
|
||||
throw new Error((payload && payload.error) || "Request failed");
|
||||
}
|
||||
|
||||
return payload ? payload.data : null;
|
||||
}
|
||||
|
||||
function escapeHtml(value) {
|
||||
return String(value == null ? "" : value)
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/\"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function setText(id, value) {
|
||||
const el = document.getElementById(id);
|
||||
|
||||
if (el) {
|
||||
el.textContent = value == null ? "-" : value;
|
||||
}
|
||||
}
|
||||
|
||||
function tableRows(rows, columns, emptyMessage) {
|
||||
if (!rows || !rows.length) {
|
||||
return '<tr><td colspan="' + columns.length + '">' + escapeHtml(emptyMessage || "No rows") + '</td></tr>';
|
||||
}
|
||||
|
||||
return rows.map(function (row) {
|
||||
return "<tr>" + columns.map(function (col) {
|
||||
let value = row[col];
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
value = value.join(", ");
|
||||
}
|
||||
|
||||
return "<td>" + escapeHtml(value) + "</td>";
|
||||
}).join("") + "</tr>";
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function countByKey(rows, keyName, valueName) {
|
||||
const result = {};
|
||||
|
||||
for (const row of rows || []) {
|
||||
result[row[keyName]] = row[valueName];
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadHealth() {
|
||||
const statusBox = document.getElementById("statusBox");
|
||||
|
||||
try {
|
||||
statusBox.className = "status ok";
|
||||
statusBox.textContent = "Loading system health...";
|
||||
|
||||
const data = await apiGet("/api/system/health-summary");
|
||||
const app = data.app || {};
|
||||
const db = data.database || {};
|
||||
const routes = data.routes || {};
|
||||
|
||||
setText("appStatus", app.status || "-");
|
||||
setText("sourceLines", Number(app.source_lines || 0).toLocaleString());
|
||||
setText("routeCount", app.route_count || 0);
|
||||
setText("duplicateRoutes", app.duplicate_route_count || 0);
|
||||
setText("middlewareCount", app.middleware_count || 0);
|
||||
setText("assetRouteCount", app.asset_route_count || 0);
|
||||
|
||||
const locationCounts = countByKey(db.location_state_counts, "asset_state", "count");
|
||||
const tableCounts = db.table_counts || {};
|
||||
const api3 = db.api3_summary || {};
|
||||
|
||||
const retention = db.retention || {};
|
||||
const rawStatus = retention.raw_status || {};
|
||||
const hourly = retention.hourly_rollups || {};
|
||||
const daily = retention.daily_rollups || {};
|
||||
const latestRetention = retention.latest_run || {};
|
||||
|
||||
setText("minerCount", tableCounts.miners || 0);
|
||||
setText("repairCount", db.repair_queue_count || 0);
|
||||
setText("shelvedCount", locationCounts.shelved || 0);
|
||||
setText("waitingCount", locationCounts.waiting_placement || 0);
|
||||
setText("api3Supported", api3.api3_supported || 0);
|
||||
setText("api3Failed", api3.api3_failed || 0);
|
||||
|
||||
setText("rawStatusRows", Number(rawStatus.raw_status_rows || 0).toLocaleString());
|
||||
setText("rawOlder30", Number(rawStatus.raw_older_than_30d || 0).toLocaleString());
|
||||
setText("hourlyRollups", Number(hourly.hourly_rows || 0).toLocaleString());
|
||||
setText("dailyRollups", Number(daily.daily_rows || 0).toLocaleString());
|
||||
setText("lastRetentionMode", latestRetention.mode || "-");
|
||||
setText("lastDeletedRows", Number(latestRetention.raw_status_rows_deleted || 0).toLocaleString());
|
||||
|
||||
const retentionDetails = document.getElementById("retentionDetails");
|
||||
if (retentionDetails) {
|
||||
retentionDetails.innerHTML =
|
||||
"Raw range: <code>" + escapeHtml(rawStatus.raw_oldest || "-") + "</code> to <code>" + escapeHtml(rawStatus.raw_newest || "-") + "</code><br>" +
|
||||
"Hourly range: <code>" + escapeHtml(hourly.hourly_oldest || "-") + "</code> to <code>" + escapeHtml(hourly.hourly_newest || "-") + "</code><br>" +
|
||||
"Daily range: <code>" + escapeHtml(daily.daily_oldest || "-") + "</code> to <code>" + escapeHtml(daily.daily_newest || "-") + "</code><br>" +
|
||||
"Latest run: <code>" + escapeHtml(latestRetention.started_at || "-") + "</code> | " + escapeHtml(latestRetention.notes || "-");
|
||||
}
|
||||
|
||||
document.getElementById("largestTablesBody").innerHTML = tableRows(
|
||||
db.largest_tables || [],
|
||||
["table_name", "total_size"],
|
||||
"No table size data"
|
||||
);
|
||||
|
||||
document.getElementById("repairStatusBody").innerHTML = tableRows(
|
||||
db.repair_status_counts || [],
|
||||
["event_status", "count"],
|
||||
"No repair events"
|
||||
);
|
||||
|
||||
document.getElementById("duplicateRoutesBody").innerHTML = tableRows(
|
||||
routes.duplicates || [],
|
||||
["method", "path", "count", "lines"],
|
||||
"No duplicate routes"
|
||||
);
|
||||
|
||||
document.getElementById("patchMarkersBody").innerHTML = tableRows(
|
||||
routes.recent_patch_markers || [],
|
||||
["line", "text"],
|
||||
"No patch markers"
|
||||
);
|
||||
|
||||
statusBox.className = "status ok";
|
||||
statusBox.textContent = "System health loaded.";
|
||||
} catch (error) {
|
||||
statusBox.className = "status bad";
|
||||
statusBox.textContent = "System health failed to load: " + (error.message || error);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById("refreshBtn").addEventListener("click", loadHealth);
|
||||
loadHealth();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@app.get("/system-health", response_class=MinerOpsSystemHealthHTMLResponse)
|
||||
def minerops_system_health_page():
|
||||
return MINEROPS_SYSTEM_HEALTH_HTML
|
||||
|
||||
|
||||
# MINEROPS_SYSTEM_HEALTH_RETENTION_V1
|
||||
|
||||
# MINEROPS_REPAIR_READY_TO_SHELF_STATUS_V1
|
||||
|
||||
Reference in new issue
Block a user