From c3501b987ed6ea858681b3f492d56796f694bfdd Mon Sep 17 00:00:00 2001 From: JC Date: Thu, 20 Aug 2026 16:06:31 +0000 Subject: [PATCH] feat: add system health diagnostics --- backend/app/config.py | 2 + backend/app/frontend/static/system-health.css | 139 ++++ backend/app/frontend/static/system-health.js | 164 ++++ backend/app/frontend/system_health_view.py | 54 ++ backend/app/frontend/views.py | 19 +- backend/app/main.py | 742 +----------------- backend/app/system_health/__init__.py | 1 + backend/app/system_health/adapters.py | 29 + backend/app/system_health/application.py | 42 + backend/app/system_health/dependencies.py | 9 + backend/app/system_health/frontend.py | 35 + backend/app/system_health/models.py | 124 +++ backend/app/system_health/repository.py | 179 +++++ backend/app/system_health/router.py | 27 + backend/app/system_health/service.py | 136 ++++ backend/tests/test_system_health.py | 304 +++++++ scripts/test-system-health.sh | 70 ++ 17 files changed, 1335 insertions(+), 741 deletions(-) create mode 100644 backend/app/frontend/static/system-health.css create mode 100644 backend/app/frontend/static/system-health.js create mode 100644 backend/app/frontend/system_health_view.py create mode 100644 backend/app/system_health/__init__.py create mode 100644 backend/app/system_health/adapters.py create mode 100644 backend/app/system_health/application.py create mode 100644 backend/app/system_health/dependencies.py create mode 100644 backend/app/system_health/frontend.py create mode 100644 backend/app/system_health/models.py create mode 100644 backend/app/system_health/repository.py create mode 100644 backend/app/system_health/router.py create mode 100644 backend/app/system_health/service.py create mode 100644 backend/tests/test_system_health.py create mode 100755 scripts/test-system-health.sh diff --git a/backend/app/config.py b/backend/app/config.py index ba7459e..32348d6 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -5,6 +5,8 @@ class Settings(BaseSettings): app_name: str = "MinerOps Console" app_env: str = "development" app_timezone: str = "America/Winnipeg" + minerops_build_commit: str | None = None + minerops_build_version: str | None = None postgres_db: str = "minerops" postgres_user: str = "minerops" diff --git a/backend/app/frontend/static/system-health.css b/backend/app/frontend/static/system-health.css new file mode 100644 index 0000000..3b5c45e --- /dev/null +++ b/backend/app/frontend/static/system-health.css @@ -0,0 +1,139 @@ +.health-toolbar, +.health-section-head { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; +} + +.health-toolbar { + margin-bottom: 12px; +} + +.health-toolbar h2, +.health-toolbar p, +.health-section h2, +.health-section h3 { + margin-top: 0; +} + +.health-toolbar p { + margin-bottom: 0; + color: var(--muted-text, #52606d); +} + +.health-notice { + border-left: 4px solid #64748b; + background: #f1f5f9; + margin: 0 0 14px; + padding: 10px 12px; +} + +.health-notice[data-state="partial"] { + border-left-color: #b45309; + background: #fff7ed; +} + +.health-notice[data-state="error"] { + border-left-color: #b91c1c; + background: #fef2f2; +} + +.health-section { + border-top: 1px solid #cbd5e1; + padding: 16px 0; +} + +.health-columns, +.health-integrity-grid, +.health-facts { + display: grid; + gap: 14px; +} + +.health-columns { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.health-integrity-grid { + grid-template-columns: repeat(3, minmax(0, 1fr)); +} + +.health-facts { + grid-template-columns: repeat(4, minmax(0, 1fr)); + margin: 0; +} + +.health-facts-compact { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.health-facts div { + min-width: 0; +} + +.health-facts dt { + color: #52606d; + font-size: 0.78rem; + font-weight: 700; +} + +.health-facts dd { + font-weight: 700; + margin: 3px 0 0; + overflow-wrap: anywhere; +} + +.health-links { + display: flex; + flex-wrap: wrap; + gap: 10px; +} + +.health-links a:focus-visible, +#healthRefresh:focus-visible { + outline: 3px solid #0f766e; + outline-offset: 3px; +} + +.table-scroll { + overflow-x: auto; +} + +.table-scroll table { + width: 100%; + border-collapse: collapse; +} + +.table-scroll th, +.table-scroll td { + border-bottom: 1px solid #dbe2e8; + padding: 7px 8px; + text-align: left; +} + +@media (max-width: 900px) { + .health-integrity-grid, + .health-facts { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@media (max-width: 640px) { + .health-toolbar, + .health-section-head { + align-items: stretch; + flex-direction: column; + } + + .health-columns, + .health-integrity-grid, + .health-facts, + .health-facts-compact { + grid-template-columns: 1fr; + } + + #healthRefresh { + width: 100%; + } +} diff --git a/backend/app/frontend/static/system-health.js b/backend/app/frontend/static/system-health.js new file mode 100644 index 0000000..f14e8fa --- /dev/null +++ b/backend/app/frontend/static/system-health.js @@ -0,0 +1,164 @@ +"use strict"; + +(function () { + const notice = document.getElementById("healthNotice"); + const content = document.getElementById("healthContent"); + const refresh = document.getElementById("healthRefresh"); + + function setText(id, value, fallback) { + const element = document.getElementById(id); + if (element) { + element.textContent = value === null || value === undefined || value === "" + ? (fallback || "Unavailable") : String(value); + } + } + + function number(value) { + return typeof value === "number" ? value.toLocaleString() : "Unavailable"; + } + + function timestamp(value) { + if (!value) return "Unavailable"; + const date = new Date(value); + return Number.isNaN(date.valueOf()) ? "Unavailable" : date.toLocaleString(); + } + + function factualAge(value) { + if (!value) return "Unavailable"; + const observed = new Date(value); + if (Number.isNaN(observed.valueOf())) return "Unavailable"; + const seconds = Math.max(0, Math.floor((Date.now() - observed.valueOf()) / 1000)); + if (seconds < 60) return seconds + " seconds ago"; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return minutes + (minutes === 1 ? " minute ago" : " minutes ago"); + const hours = Math.floor(minutes / 60); + if (hours < 24) return hours + (hours === 1 ? " hour ago" : " hours ago"); + const days = Math.floor(hours / 24); + return days + (days === 1 ? " day ago" : " days ago"); + } + + function replaceFacts(id, facts) { + const list = document.getElementById(id); + list.replaceChildren(); + for (const [label, value] of facts) { + const row = document.createElement("div"); + const term = document.createElement("dt"); + const detail = document.createElement("dd"); + term.textContent = label; + detail.textContent = value; + row.append(term, detail); + list.append(row); + } + } + + function render(payload) { + const data = payload.data; + const app = data.application; + const postgres = data.postgresql; + const migration = postgres.migration; + const telemetry = postgres.telemetry; + setText("healthAppStatus", app.status); + setText("healthEnvironment", app.environment); + setText("healthBuildCommit", app.build_commit); + setText("healthBuildVersion", app.build_version); + setText("healthServerTime", timestamp(app.server_time)); + setText("healthMigrationState", migration.state); + setText("healthCurrentRevision", migration.current_revision); + setText("healthExpectedRevision", migration.expected_revision); + setText("healthPostgresqlStatus", postgres.status); + setText("healthRedisStatus", data.redis.status); + setText("healthSnapshotCount", telemetry ? number(telemetry.snapshot_count) : null); + setText("healthOldestObservation", telemetry ? timestamp(telemetry.oldest_observed_at) : null); + setText("healthLatestObservation", telemetry ? timestamp(telemetry.latest_observed_at) : null); + setText("healthObservationAge", telemetry ? factualAge(telemetry.latest_observed_at) : null); + + const fleet = postgres.fleet; + replaceFacts("healthFleetFacts", fleet ? [ + ["Total miners", number(fleet.total_miners)], + ["Without Asset ID", number(fleet.miners_without_asset_id)], + ["Unmapped", number(fleet.unmapped_miners)], + ["In multiple positions", number(fleet.miners_in_multiple_positions)], + ["Duplicate current MAC groups", number(fleet.duplicate_current_mac_groups)] + ] : [["Diagnostics", "Unavailable"]]); + const locations = postgres.locations; + replaceFacts("healthLocationFacts", locations ? [ + ["Sites", number(locations.sites)], ["Rooms", number(locations.rooms)], + ["Positions", number(locations.positions)], + ["Occupied", number(locations.occupied_positions)], + ["Empty", number(locations.empty_positions)], + ["Legacy unscoped", number(locations.unscoped_legacy_positions)], + ["Expected MAC mismatches", number(locations.expected_mac_mismatches)] + ] : [["Diagnostics", "Unavailable"]]); + const repairs = postgres.repairs; + replaceFacts("healthRepairFacts", repairs ? [ + ["Active modular repairs", number(repairs.active_modular_repairs)], + ["Duplicate active groups", number(repairs.duplicate_active_repair_groups)], + ["Legacy repair only", number(repairs.legacy_repair_only_miners)], + ["Lifecycle repair without active repair", number(repairs.lifecycle_repair_without_active_repair)], + ["Active repair lifecycle mismatches", number(repairs.active_repair_lifecycle_mismatches)], + ["Waiting placement", number(repairs.waiting_placement_records)], + ["Repair events", number(repairs.repair_events)], + ["Repair parts", number(repairs.repair_parts)] + ] : [["Diagnostics", "Unavailable"]]); + const retention = postgres.retention; + replaceFacts("healthRetentionFacts", retention ? [ + ["Raw telemetry retention", retention.raw_retention_enabled ? "Enabled" : "Not enabled"], + ["Hourly rollups", retention.hourly.table_present ? "Present; manual population" : "Unavailable"], + ["Hourly latest bucket", timestamp(retention.hourly.latest_bucket)], + ["Daily rollups", retention.daily.table_present ? "Present; manual population" : "Unavailable"], + ["Daily latest bucket", timestamp(retention.daily.latest_bucket)], + ["Latest recorded run", timestamp(retention.latest_recorded_run_at)], + ["Latest run mode", retention.latest_recorded_run_mode || "None recorded"], + ["Completeness", "Not guaranteed"] + ] : [["Diagnostics", "Unavailable"]]); + + const tableBody = document.getElementById("healthTableCounts"); + tableBody.replaceChildren(); + const rows = postgres.table_counts || []; + for (const table of rows) { + const row = document.createElement("tr"); + const name = document.createElement("th"); + const count = document.createElement("td"); + name.scope = "row"; + name.textContent = table.name; + count.textContent = number(table.row_count); + row.append(name, count); + tableBody.append(row); + } + if (!rows.length) { + const row = document.createElement("tr"); + const cell = document.createElement("td"); + cell.colSpan = 2; + cell.textContent = "Table diagnostics unavailable."; + row.append(cell); + tableBody.append(row); + } + content.hidden = false; + notice.dataset.state = payload.meta.partial ? "partial" : "ready"; + notice.textContent = payload.meta.partial + ? "Diagnostics loaded with unavailable sections: " + payload.meta.unavailable_sections.join(", ") + "." + : "Diagnostics refreshed at " + timestamp(payload.meta.generated_at) + "."; + } + + async function load() { + refresh.disabled = true; + notice.dataset.state = "loading"; + notice.textContent = "Loading diagnostics."; + try { + const response = await fetch("/api/system/health-summary", { + credentials: "same-origin", headers: {"Accept": "application/json"} + }); + const payload = await response.json().catch(function () { return {}; }); + if (!response.ok) throw new Error("System Health request failed"); + render(payload); + } catch (error) { + notice.dataset.state = "error"; + notice.textContent = "System Health diagnostics could not be loaded."; + } finally { + refresh.disabled = false; + } + } + + refresh.addEventListener("click", load); + load(); +}()); diff --git a/backend/app/frontend/system_health_view.py b/backend/app/frontend/system_health_view.py new file mode 100644 index 0000000..c55c34e --- /dev/null +++ b/backend/app/frontend/system_health_view.py @@ -0,0 +1,54 @@ +"""Semantic content for the Internal Alpha System Health page.""" + + +def system_health_content() -> str: + return """ +
+

Operations diagnostics

+

Availability, data facts, integrity counts, and maintenance state.

+ +
+

Loading diagnostics.

+ +""" diff --git a/backend/app/frontend/views.py b/backend/app/frontend/views.py index 2d6a081..87fb8e3 100644 --- a/backend/app/frontend/views.py +++ b/backend/app/frontend/views.py @@ -5,6 +5,7 @@ from typing import Callable from app.frontend.miner_detail_view import miner_detail_content from app.frontend.monitoring_trends_view import monitoring_trends_content +from app.frontend.system_health_view import system_health_content NAVIGATION = ( @@ -13,6 +14,7 @@ NAVIGATION = ( ("analytics", "Trends", "/monitoring/analytics", "miners.view"), ("repairs", "Repair Center", "/operations/repairs", "repairs.view"), ("miners", "Miners", "/assets/miners", "miners.view"), + ("system_health", "System Health", "/system-health", "system_health.view"), ) PAGE_DETAILS = { @@ -46,6 +48,11 @@ PAGE_DETAILS = { "Miner Detail", "Current identity, operating facts, location, and authorized history.", ), + "system_health": ( + "Administration / Diagnostics", + "System Health", + "Inspect application services, operational data, and maintenance facts.", + ), } @@ -455,7 +462,9 @@ def render_app_page( else '' ) content = ( - miner_detail_content() + system_health_content() + if view == "system_health" + else miner_detail_content() if view == "miner_detail" else monitoring_trends_content(has_permission("locations.view")) if view == "analytics" @@ -474,7 +483,9 @@ def render_app_page( ) ) page_script = ( - '' + '' + if view == "system_health" + else '' if view == "miner_detail" else '' if view == "analytics" @@ -489,7 +500,9 @@ def render_app_page( else "" ) page_style = ( - '' + '' + if view == "system_health" + else '' if view == "analytics" else "" ) object_attributes = "" diff --git a/backend/app/main.py b/backend/app/main.py index 21aa961..239462a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -32,6 +32,8 @@ from app.components.router import router as components_router from app.miner_detail.router import router as miner_detail_router from app.monitoring.router import router as monitoring_router from app.monitoring.frontend import router as monitoring_frontend_router +from app.system_health.frontend import router as system_health_frontend_router +from app.system_health.router import router as system_health_router from app.repairs.exceptions import ( ControlBoardReplacementActorRequired, InvalidMacAddress, @@ -67,6 +69,8 @@ app.include_router(dashboard_router) app.include_router(miner_detail_router) app.include_router(monitoring_frontend_router) app.include_router(monitoring_router) +app.include_router(system_health_frontend_router) +app.include_router(system_health_router) install_identity_middleware(app) @@ -34663,744 +34667,6 @@ async def minerops_inject_repair_history_modal(request, call_next): -# MINEROPS_SYSTEM_HEALTH_V1 -from fastapi.responses import HTMLResponse as MinerOpsSystemHealthHTMLResponse -from pathlib import Path as MinerOpsSystemPath - - -@app.get("/api/system/health-summary") -def minerops_system_health_summary( - current_user: dict = Depends(get_current_user), -): - import re - from collections import Counter - - ensure_location_tables() - ensure_maintenance_tables() - - source_path = MinerOpsSystemPath(__file__) - source_text = source_path.read_text(errors="ignore") - source_lines = source_text.splitlines() - - route_re = re.compile(r'^\s*@app\.(get|post|patch|delete|put)\("([^"]+)"') - routes = [] - - for line_no, line in enumerate(source_lines, start=1): - match = route_re.search(line) - - if match: - routes.append( - { - "method": match.group(1).upper(), - "path": match.group(2), - "line": line_no, - } - ) - - route_counter = Counter((r["method"], r["path"]) for r in routes) - duplicate_routes = [] - - for (method, route_path), count in sorted(route_counter.items(), key=lambda item: (item[0][1], item[0][0])): - if count > 1: - duplicate_routes.append( - { - "method": method, - "path": route_path, - "count": count, - "lines": [ - r["line"] - for r in routes - if r["method"] == method and r["path"] == route_path - ], - } - ) - - middleware_lines = [ - line_no - for line_no, line in enumerate(source_lines, start=1) - if '@app.middleware("http")' in line - ] - - asset_routes = [ - r - for r in routes - if str(r["path"]).startswith("/assets/") - ] - - patch_markers = [] - - for line_no, line in enumerate(source_lines, start=1): - stripped = line.strip() - - if stripped.startswith("# MINEROPS_") or stripped.startswith("# Step "): - patch_markers.append( - { - "line": line_no, - "text": stripped, - } - ) - - conn = None - - try: - conn = get_db_connection() - - with conn.cursor(row_factory=psycopg.rows.dict_row) as cur: - cur.execute( - """ - SELECT - relname AS table_name, - pg_total_relation_size(relid) AS total_bytes, - pg_size_pretty(pg_total_relation_size(relid)) AS total_size - FROM pg_catalog.pg_statio_user_tables - ORDER BY pg_total_relation_size(relid) DESC - LIMIT 15; - """ - ) - largest_tables = [dict(row) for row in cur.fetchall()] - - cur.execute( - """ - SELECT - relname AS table_name, - n_live_tup::bigint AS estimated_rows, - n_dead_tup::bigint AS estimated_dead_rows, - last_vacuum, - last_autovacuum, - last_analyze, - last_autoanalyze - FROM pg_stat_user_tables - ORDER BY n_live_tup DESC - LIMIT 20; - """ - ) - row_estimates = [dict(row) for row in cur.fetchall()] - - table_counts = {} - - for table_name in [ - "miners", - "location_positions", - "miner_location_states", - "miner_maintenance_events", - "miner_hashboards", - "miner_network_identities", - "minerops_agent_command_queue", - ]: - try: - cur.execute(f"SELECT COUNT(*) AS row_count FROM {table_name};") - table_counts[table_name] = cur.fetchone()["row_count"] - except Exception as exc: - table_counts[table_name] = f"error: {exc}" - - cur.execute( - """ - SELECT - COALESCE(status, 'unknown') AS status, - COUNT(*) AS count - FROM miners - GROUP BY COALESCE(status, 'unknown') - ORDER BY status; - """ - ) - miner_status_counts = [dict(row) for row in cur.fetchall()] - - cur.execute( - """ - SELECT - COALESCE(asset_state, 'unknown') AS asset_state, - COUNT(*) AS count - FROM miner_location_states - GROUP BY COALESCE(asset_state, 'unknown') - ORDER BY asset_state; - """ - ) - location_state_counts = [dict(row) for row in cur.fetchall()] - - cur.execute( - """ - SELECT - COALESCE(event_status, 'unknown') AS event_status, - COUNT(*) AS count - FROM miner_maintenance_events - WHERE event_type = 'repair' - GROUP BY COALESCE(event_status, 'unknown') - ORDER BY event_status; - """ - ) - repair_status_counts = [dict(row) for row in cur.fetchall()] - - cur.execute( - """ - SELECT - COUNT(*) FILTER (WHERE api3_supported IS TRUE) AS api3_supported, - COUNT(*) FILTER (WHERE api3_supported IS FALSE) AS api3_failed, - COUNT(*) FILTER (WHERE api3_supported IS NULL) AS api3_unknown, - COUNT(*) FILTER (WHERE power_sn IS NOT NULL AND power_sn <> '') AS miners_with_power_sn, - COUNT(*) FILTER (WHERE miner_sn IS NOT NULL AND miner_sn <> '') AS miners_with_miner_sn - FROM miners; - """ - ) - api3_summary = dict(cur.fetchone()) - - cur.execute( - """ - SELECT - COUNT(*) AS repair_queue_count - FROM miner_location_states - WHERE asset_state = 'repair'; - """ - ) - repair_queue_count = cur.fetchone()["repair_queue_count"] - - cur.execute( - """ - SELECT - COUNT(*) AS raw_status_rows, - MIN(observed_at) AS raw_oldest, - MAX(observed_at) AS raw_newest, - COUNT(*) FILTER (WHERE observed_at < now() - interval '30 days') AS raw_older_than_30d - FROM miner_status_snapshots; - """ - ) - raw_status_retention = dict(cur.fetchone()) - - cur.execute( - """ - SELECT - COUNT(*) AS hourly_rows, - MIN(bucket_at) AS hourly_oldest, - MAX(bucket_at) AS hourly_newest - FROM miner_status_hourly_rollups; - """ - ) - hourly_retention = dict(cur.fetchone()) - - cur.execute( - """ - SELECT - COUNT(*) AS daily_rows, - MIN(bucket_date) AS daily_oldest, - MAX(bucket_date) AS daily_newest - FROM miner_status_daily_rollups; - """ - ) - daily_retention = dict(cur.fetchone()) - - cur.execute( - """ - SELECT - mode, - raw_status_retention_days, - raw_status_rows_eligible_delete, - raw_status_rows_deleted, - hourly_rollup_rows_after, - daily_rollup_rows_after, - notes, - started_at, - finished_at - FROM minerops_retention_runs - ORDER BY started_at DESC - LIMIT 1; - """ - ) - latest_retention_run = cur.fetchone() - latest_retention_run = dict(latest_retention_run) if latest_retention_run else None - - return api_response( - data={ - "app": { - "status": "ok", - "source_lines": len(source_lines), - "source_size_bytes": source_path.stat().st_size, - "route_count": len(routes), - "duplicate_route_count": len(duplicate_routes), - "middleware_count": len(middleware_lines), - "asset_route_count": len(asset_routes), - }, - "routes": { - "duplicates": duplicate_routes, - "middleware_lines": middleware_lines, - "asset_routes": asset_routes, - "recent_patch_markers": patch_markers[-40:], - }, - "database": { - "largest_tables": largest_tables, - "row_estimates": row_estimates, - "table_counts": table_counts, - "miner_status_counts": miner_status_counts, - "location_state_counts": location_state_counts, - "repair_status_counts": repair_status_counts, - "api3_summary": api3_summary, - "repair_queue_count": repair_queue_count, - "retention": { - "raw_status": raw_status_retention, - "hourly_rollups": hourly_retention, - "daily_rollups": daily_retention, - "latest_run": latest_retention_run, - }, - }, - } - ) - - finally: - if conn is not None: - conn.close() - - -MINEROPS_SYSTEM_HEALTH_HTML = r""" - - - - MinerOps System Health - - - - -
- Dashboard - Repairs - Locations - System Health -
- -
-
-

System Health

-
Quick operational view of MinerOps app size, routes, database tables, repairs, and location state.
-
- -
- -
Loading system health...
- -
-

App Health

-
-
Status
-
-
main.py lines
-
-
Routes
-
-
Duplicate routes
-
-
Middleware
-
-
Assets
-
-
-
- -
-

Fleet / Repair State

-
-
Miners
-
-
In Repair
-
-
Shelved
-
-
Waiting Placement
-
-
API3 Supported
-
-
API3 Failed
-
-
-
- -
-

Status Retention

-
-
Raw Status Rows
-
-
Older Than 30 Days
-
-
Hourly Rollups
-
-
Daily Rollups
-
-
Last Retention Mode
-
-
Last Deleted Rows
-
-
-
Loading retention details...
-
- -
-
-

Largest Tables

-
- - - -
TableSize
Loading...
-
-
- -
-

Repair Status Counts

-
- - - -
StatusCount
Loading...
-
-
-
- -
-
-

Duplicate Routes

-
- - - -
MethodPathCountLines
Loading...
-
-
- -
-

Recent Patch Markers

-
- - - -
LineMarker
Loading...
-
-
-
- - - - -""" - - -@app.get("/system-health", response_class=MinerOpsSystemHealthHTMLResponse) -def minerops_system_health_page(): - return MINEROPS_SYSTEM_HEALTH_HTML - - # MINEROPS_SYSTEM_HEALTH_RETENTION_V1 # MINEROPS_REPAIR_READY_TO_SHELF_STATUS_V1 diff --git a/backend/app/system_health/__init__.py b/backend/app/system_health/__init__.py new file mode 100644 index 0000000..ba3bcfe --- /dev/null +++ b/backend/app/system_health/__init__.py @@ -0,0 +1 @@ +"""System Health diagnostics domain.""" diff --git a/backend/app/system_health/adapters.py b/backend/app/system_health/adapters.py new file mode 100644 index 0000000..57f1f2d --- /dev/null +++ b/backend/app/system_health/adapters.py @@ -0,0 +1,29 @@ +"""External service adapters for System Health.""" + +from collections.abc import Callable + +import redis + +from app.config import settings + + +RedisFactory = Callable[[], redis.Redis] + + +def default_redis_factory() -> redis.Redis: + return redis.Redis.from_url( + settings.redis_url, + socket_connect_timeout=2, + socket_timeout=2, + ) + + +class RedisHealthAdapter: + def __init__(self, factory: RedisFactory | None = None): + self._factory = factory or default_redis_factory + + def ping(self) -> bool: + try: + return bool(self._factory().ping()) + except Exception: + return False diff --git a/backend/app/system_health/application.py b/backend/app/system_health/application.py new file mode 100644 index 0000000..82f458b --- /dev/null +++ b/backend/app/system_health/application.py @@ -0,0 +1,42 @@ +"""Startup-cached, non-secret application metadata.""" + +from dataclasses import dataclass +from pathlib import Path + +from alembic.config import Config +from alembic.script import ScriptDirectory + +from app.config import settings + + +def _clean_build_value(value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned[:128] if cleaned else None + + +def _expected_migration_head() -> str | None: + try: + backend_root = Path(__file__).resolve().parents[2] + config = Config(str(backend_root / "alembic.ini")) + config.set_main_option("script_location", str(backend_root / "alembic")) + return ScriptDirectory.from_config(config).get_current_head() + except Exception: + return None + + +@dataclass(frozen=True) +class ApplicationMetadata: + environment: str + build_commit: str | None + build_version: str | None + expected_migration_head: str | None + + +APPLICATION_METADATA = ApplicationMetadata( + environment=settings.app_env, + build_commit=_clean_build_value(settings.minerops_build_commit), + build_version=_clean_build_value(settings.minerops_build_version), + expected_migration_head=_expected_migration_head(), +) diff --git a/backend/app/system_health/dependencies.py b/backend/app/system_health/dependencies.py new file mode 100644 index 0000000..f9c49d8 --- /dev/null +++ b/backend/app/system_health/dependencies.py @@ -0,0 +1,9 @@ +"""Dependency construction for System Health.""" + +from app.system_health.adapters import RedisHealthAdapter +from app.system_health.repository import SystemHealthRepository +from app.system_health.service import SystemHealthService + + +def get_system_health_service() -> SystemHealthService: + return SystemHealthService(SystemHealthRepository(), RedisHealthAdapter()) diff --git a/backend/app/system_health/frontend.py b/backend/app/system_health/frontend.py new file mode 100644 index 0000000..55336cc --- /dev/null +++ b/backend/app/system_health/frontend.py @@ -0,0 +1,35 @@ +"""Authenticated frontend routes for System Health.""" + +from pathlib import Path +from typing import Annotated + +from fastapi import APIRouter, Depends +from fastapi.responses import HTMLResponse, Response + +from app.frontend.views import render_app_page +from app.identity import require_permission, user_has_permission + + +router = APIRouter(tags=["frontend"]) +STATIC_ROOT = Path(__file__).parents[1] / "frontend" / "static" +SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))] + + +@router.get("/assets/minerops-system-health.css", include_in_schema=False) +def system_health_styles() -> Response: + return Response((STATIC_ROOT / "system-health.css").read_text(encoding="utf-8"), + media_type="text/css", headers={"Cache-Control": "no-store"}) + + +@router.get("/assets/minerops-system-health.js", include_in_schema=False) +def system_health_script() -> Response: + return Response((STATIC_ROOT / "system-health.js").read_text(encoding="utf-8"), + media_type="text/javascript", headers={"Cache-Control": "no-store"}) + + +@router.get("/system-health", response_class=HTMLResponse, include_in_schema=False) +def system_health_page(current_user: SystemHealthUser) -> str: + return render_app_page( + "system_health", current_user, + lambda permission: user_has_permission(current_user, permission), + ) diff --git a/backend/app/system_health/models.py b/backend/app/system_health/models.py new file mode 100644 index 0000000..4f18768 --- /dev/null +++ b/backend/app/system_health/models.py @@ -0,0 +1,124 @@ +"""Typed contracts for the System Health overview.""" + +from datetime import date, datetime +from typing import Literal + +from pydantic import BaseModel, Field + + +Availability = Literal["available", "unavailable"] +MigrationState = Literal["current", "mismatch", "unavailable"] + + +class ApplicationDiagnostics(BaseModel): + status: Literal["available"] = "available" + environment: str + build_commit: str | None = None + build_version: str | None = None + server_time: datetime + expected_migration_head: str | None = None + + +class MigrationDiagnostics(BaseModel): + state: MigrationState + current_revision: str | None = None + expected_revision: str | None = None + + +class TableCount(BaseModel): + name: str + row_count: int + + +class TableSize(BaseModel): + name: str + total_bytes: int + + +class TelemetryDiagnostics(BaseModel): + snapshot_count: int + oldest_observed_at: datetime | None = None + latest_observed_at: datetime | None = None + + +class FleetIntegrityDiagnostics(BaseModel): + total_miners: int + miners_without_asset_id: int + unmapped_miners: int + miners_in_multiple_positions: int + duplicate_current_mac_groups: int + + +class LocationIntegrityDiagnostics(BaseModel): + sites: int + rooms: int + positions: int + occupied_positions: int + empty_positions: int + unscoped_legacy_positions: int + expected_mac_mismatches: int + + +class RepairIntegrityDiagnostics(BaseModel): + active_modular_repairs: int + duplicate_active_repair_groups: int + legacy_repair_only_miners: int + lifecycle_repair_without_active_repair: int + active_repair_lifecycle_mismatches: int + waiting_placement_records: int + repair_events: int + repair_parts: int + + +class RollupDiagnostics(BaseModel): + table_present: bool + row_count: int | None = None + latest_bucket: datetime | date | None = None + population: Literal["manual"] = "manual" + + +class RetentionDiagnostics(BaseModel): + raw_retention_enabled: Literal[False] = False + raw_retention_mode: Literal["not_enabled"] = "not_enabled" + hourly: RollupDiagnostics + daily: RollupDiagnostics + latest_recorded_run_at: datetime | None = None + latest_recorded_run_mode: str | None = None + completeness: Literal["not_guaranteed"] = "not_guaranteed" + + +class PostgreSQLDiagnostics(BaseModel): + status: Availability + migration: MigrationDiagnostics + table_counts: list[TableCount] = Field(default_factory=list) + largest_tables: list[TableSize] = Field(default_factory=list) + telemetry: TelemetryDiagnostics | None = None + fleet: FleetIntegrityDiagnostics | None = None + locations: LocationIntegrityDiagnostics | None = None + repairs: RepairIntegrityDiagnostics | None = None + retention: RetentionDiagnostics | None = None + + +class RedisDiagnostics(BaseModel): + status: Availability + + +class SystemHealthOverviewData(BaseModel): + application: ApplicationDiagnostics + postgresql: PostgreSQLDiagnostics + redis: RedisDiagnostics + + +class SystemHealthOverviewMeta(BaseModel): + generated_at: datetime + partial: bool + unavailable_sections: list[str] = Field(default_factory=list) + frontend_request_count: Literal[1] = 1 + postgresql_statement_count: Literal[3] = 3 + redis_call_count: Literal[1] = 1 + + +class SystemHealthOverviewResponse(BaseModel): + data: SystemHealthOverviewData + meta: SystemHealthOverviewMeta + error: None = None diff --git a/backend/app/system_health/repository.py b/backend/app/system_health/repository.py new file mode 100644 index 0000000..d3edc40 --- /dev/null +++ b/backend/app/system_health/repository.py @@ -0,0 +1,179 @@ +"""PostgreSQL diagnostics for System Health.""" + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +import psycopg +from psycopg.rows import dict_row + +from app.config import settings + + +ConnectionFactory = Callable[[], psycopg.Connection] + + +def default_connection_factory() -> psycopg.Connection: + return psycopg.connect( + host=settings.postgres_host, + port=settings.postgres_port, + dbname=settings.postgres_db, + user=settings.postgres_user, + password=settings.postgres_password, + connect_timeout=3, + ) + + +MIGRATION_SQL = """ +SELECT clock_timestamp() AS database_time, + (SELECT version_num FROM alembic_version LIMIT 1) AS current_revision +""" + +OPERATIONS_SQL = """ +WITH telemetry AS ( + SELECT count(*)::bigint AS snapshot_count, + min(observed_at) AS oldest_observed_at, + max(observed_at) AS latest_observed_at + FROM miner_status_snapshots +), +hourly AS ( + SELECT count(*)::bigint AS row_count, max(bucket_at) AS latest_bucket + FROM miner_status_hourly_rollups +), +daily AS ( + SELECT count(*)::bigint AS row_count, max(bucket_date) AS latest_bucket + FROM miner_status_daily_rollups +), +latest_retention AS ( + SELECT mode, started_at FROM minerops_retention_runs + ORDER BY started_at DESC LIMIT 1 +), +largest AS ( + SELECT coalesce(jsonb_agg(to_jsonb(sized) ORDER BY sized.total_bytes DESC), '[]'::jsonb) AS rows + FROM ( + SELECT relname AS name, pg_total_relation_size(relid)::bigint AS total_bytes + FROM pg_catalog.pg_statio_user_tables + WHERE relname = ANY(%s) + ORDER BY pg_total_relation_size(relid) DESC, relname + LIMIT 8 + ) AS sized +) +SELECT + (SELECT count(*) FROM miners)::bigint AS miners, + (SELECT count(*) FROM sites)::bigint AS sites, + (SELECT count(*) FROM rooms)::bigint AS rooms, + (SELECT count(*) FROM location_positions)::bigint AS location_positions, + (SELECT count(*) FROM repairs)::bigint AS repairs, + (SELECT count(*) FROM repair_events)::bigint AS repair_events, + (SELECT count(*) FROM repair_parts)::bigint AS repair_parts, + telemetry.snapshot_count, telemetry.oldest_observed_at, + telemetry.latest_observed_at, + to_regclass('public.miner_status_hourly_rollups') IS NOT NULL AS hourly_present, + hourly.row_count AS hourly_rows, hourly.latest_bucket AS hourly_latest, + to_regclass('public.miner_status_daily_rollups') IS NOT NULL AS daily_present, + daily.row_count AS daily_rows, daily.latest_bucket AS daily_latest, + latest_retention.mode AS retention_mode, + latest_retention.started_at AS retention_started_at, + largest.rows AS largest_tables +FROM telemetry CROSS JOIN hourly CROSS JOIN daily CROSS JOIN largest +LEFT JOIN latest_retention ON true +""" + +INTEGRITY_SQL = """ +WITH position_counts AS ( + SELECT miner_id, count(*) AS position_count + FROM location_positions WHERE miner_id IS NOT NULL GROUP BY miner_id +), +active_repairs AS ( + SELECT id, miner_id FROM repairs + WHERE status IN ('OPEN', 'IN_PROGRESS', 'WAITING_FOR_PARTS', 'READY_FOR_TEST') +), +active_repair_groups AS ( + SELECT miner_id FROM active_repairs GROUP BY miner_id HAVING count(*) > 1 +), +normalized_macs AS ( + SELECT lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g')) AS mac + FROM miners WHERE current_mac IS NOT NULL AND trim(current_mac) <> '' + GROUP BY lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g')) + HAVING count(*) > 1 +) +SELECT + (SELECT count(*) FROM miners WHERE asset_tag IS NULL OR trim(asset_tag) = '')::bigint + AS miners_without_asset_id, + (SELECT count(*) FROM miners m WHERE NOT EXISTS ( + SELECT 1 FROM location_positions p WHERE p.miner_id = m.id + ))::bigint AS unmapped_miners, + (SELECT count(*) FROM position_counts WHERE position_count > 1)::bigint + AS miners_in_multiple_positions, + (SELECT count(*) FROM normalized_macs)::bigint AS duplicate_current_mac_groups, + (SELECT count(*) FROM location_positions WHERE room_id IS NULL)::bigint + AS unscoped_legacy_positions, + (SELECT count(*) FROM location_positions p JOIN miners m ON m.id = p.miner_id + WHERE p.expected_mac IS NOT NULL AND trim(p.expected_mac) <> '' + AND m.current_mac IS NOT NULL AND trim(m.current_mac) <> '' + AND lower(regexp_replace(trim(p.expected_mac), '[^0-9A-Fa-f]', '', 'g')) + <> lower(regexp_replace(trim(m.current_mac), '[^0-9A-Fa-f]', '', 'g')) + )::bigint AS expected_mac_mismatches, + (SELECT count(*) FROM location_positions WHERE miner_id IS NOT NULL)::bigint + AS occupied_positions, + (SELECT count(*) FROM location_positions WHERE miner_id IS NULL)::bigint + AS empty_positions, + (SELECT count(*) FROM active_repairs)::bigint AS active_modular_repairs, + (SELECT count(*) FROM active_repair_groups)::bigint AS duplicate_active_repair_groups, + (SELECT count(*) FROM miner_location_states state + WHERE state.asset_state = 'repair' AND NOT EXISTS ( + SELECT 1 FROM active_repairs repair WHERE repair.miner_id = state.miner_id + ))::bigint AS legacy_repair_only_miners, + (SELECT count(*) FROM miners m WHERE m.lifecycle_status = 'in_repair' + AND NOT EXISTS (SELECT 1 FROM active_repairs r WHERE r.miner_id = m.id))::bigint + AS lifecycle_repair_without_active_repair, + (SELECT count(*) FROM active_repairs r JOIN miners m ON m.id = r.miner_id + WHERE m.lifecycle_status <> 'in_repair')::bigint + AS active_repair_lifecycle_mismatches, + (SELECT count(*) FROM miner_location_states WHERE asset_state = 'waiting_placement')::bigint + AS waiting_placement_records +""" + + +RELEVANT_TABLES = [ + "miner_status_snapshots", "repair_events", "repair_parts", "repairs", + "location_positions", "miners", "miner_status_hourly_rollups", + "miner_status_daily_rollups", +] + + +class SystemHealthRepository: + def __init__( + self, + connection_factory: ConnectionFactory | None = None, + *, + connection: psycopg.Connection | None = None, + ): + self._connection_factory = connection_factory or default_connection_factory + self._connection = connection + + @contextmanager + def _connection_scope(self) -> Iterator[psycopg.Connection]: + if self._connection is not None: + yield self._connection + return + connection = self._connection_factory() + try: + yield connection + finally: + connection.close() + + def read_overview(self) -> dict[str, Any]: + with self._connection_scope() as connection: + with connection.cursor(row_factory=dict_row) as cur: + cur.execute(MIGRATION_SQL) + migration = dict(cur.fetchone()) + cur.execute(OPERATIONS_SQL, (RELEVANT_TABLES,)) + operations = dict(cur.fetchone()) + cur.execute(INTEGRITY_SQL) + integrity = dict(cur.fetchone()) + return { + "migration": migration, + "operations": operations, + "integrity": integrity, + } diff --git a/backend/app/system_health/router.py b/backend/app/system_health/router.py new file mode 100644 index 0000000..2be2d2c --- /dev/null +++ b/backend/app/system_health/router.py @@ -0,0 +1,27 @@ +"""HTTP boundary for the System Health overview.""" + +from typing import Annotated + +from fastapi import APIRouter, Depends, Response + +from app.identity import require_permission +from app.system_health.dependencies import get_system_health_service +from app.system_health.models import SystemHealthOverviewResponse +from app.system_health.service import SystemHealthService + + +router = APIRouter(prefix="/api/system", tags=["system-health"]) +SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))] +SystemHealthServiceDependency = Annotated[ + SystemHealthService, Depends(get_system_health_service) +] + + +@router.get("/health-summary", response_model=SystemHealthOverviewResponse) +def get_system_health_overview( + current_user: SystemHealthUser, + service: SystemHealthServiceDependency, + response: Response, +) -> SystemHealthOverviewResponse: + response.headers["Cache-Control"] = "no-store" + return service.get_overview() diff --git a/backend/app/system_health/service.py b/backend/app/system_health/service.py new file mode 100644 index 0000000..11a419f --- /dev/null +++ b/backend/app/system_health/service.py @@ -0,0 +1,136 @@ +"""Composition and failure semantics for System Health diagnostics.""" + +from datetime import datetime, timezone + +from app.system_health.adapters import RedisHealthAdapter +from app.system_health.application import APPLICATION_METADATA, ApplicationMetadata +from app.system_health.models import ( + ApplicationDiagnostics, FleetIntegrityDiagnostics, LocationIntegrityDiagnostics, + MigrationDiagnostics, PostgreSQLDiagnostics, RedisDiagnostics, + RepairIntegrityDiagnostics, RetentionDiagnostics, RollupDiagnostics, + SystemHealthOverviewData, SystemHealthOverviewMeta, + SystemHealthOverviewResponse, TableCount, TableSize, TelemetryDiagnostics, +) +from app.system_health.repository import SystemHealthRepository + + +class SystemHealthService: + def __init__(self, repository: SystemHealthRepository, + redis_adapter: RedisHealthAdapter, + metadata: ApplicationMetadata = APPLICATION_METADATA): + self.repository = repository + self.redis_adapter = redis_adapter + self.metadata = metadata + + def get_overview(self) -> SystemHealthOverviewResponse: + generated_at = datetime.now(timezone.utc) + unavailable = [] + try: + postgresql = self._postgresql_diagnostics( + self.repository.read_overview() + ) + except Exception: + unavailable.append("postgresql") + postgresql = PostgreSQLDiagnostics( + status="unavailable", + migration=MigrationDiagnostics( + state="unavailable", + expected_revision=self.metadata.expected_migration_head, + ), + ) + redis_available = self.redis_adapter.ping() + if not redis_available: + unavailable.append("redis") + return SystemHealthOverviewResponse( + data=SystemHealthOverviewData( + application=ApplicationDiagnostics( + environment=self.metadata.environment, + build_commit=self.metadata.build_commit, + build_version=self.metadata.build_version, + server_time=generated_at, + expected_migration_head=self.metadata.expected_migration_head, + ), + postgresql=postgresql, + redis=RedisDiagnostics( + status="available" if redis_available else "unavailable" + ), + ), + meta=SystemHealthOverviewMeta( + generated_at=generated_at, + partial=bool(unavailable), + unavailable_sections=unavailable, + ), + ) + + def _postgresql_diagnostics(self, payload: dict) -> PostgreSQLDiagnostics: + migration = payload["migration"] + operations = payload["operations"] + integrity = payload["integrity"] + current = migration.get("current_revision") + expected = self.metadata.expected_migration_head + state = ( + "unavailable" if not current or not expected + else "current" if current == expected else "mismatch" + ) + counts = [ + TableCount(name=name, row_count=int(operations[name])) + for name in ( + "miners", "sites", "rooms", "location_positions", "repairs", + "repair_events", "repair_parts", "snapshot_count", + ) + ] + return PostgreSQLDiagnostics( + status="available", + migration=MigrationDiagnostics( + state=state, current_revision=current, expected_revision=expected + ), + table_counts=counts, + largest_tables=[ + TableSize(name=row["name"], total_bytes=int(row["total_bytes"])) + for row in operations.get("largest_tables") or [] + ], + telemetry=TelemetryDiagnostics( + snapshot_count=int(operations["snapshot_count"]), + oldest_observed_at=operations.get("oldest_observed_at"), + latest_observed_at=operations.get("latest_observed_at"), + ), + fleet=FleetIntegrityDiagnostics( + total_miners=int(operations["miners"]), + miners_without_asset_id=int(integrity["miners_without_asset_id"]), + unmapped_miners=int(integrity["unmapped_miners"]), + miners_in_multiple_positions=int(integrity["miners_in_multiple_positions"]), + duplicate_current_mac_groups=int(integrity["duplicate_current_mac_groups"]), + ), + locations=LocationIntegrityDiagnostics( + sites=int(operations["sites"]), rooms=int(operations["rooms"]), + positions=int(operations["location_positions"]), + occupied_positions=int(integrity["occupied_positions"]), + empty_positions=int(integrity["empty_positions"]), + unscoped_legacy_positions=int(integrity["unscoped_legacy_positions"]), + expected_mac_mismatches=int(integrity["expected_mac_mismatches"]), + ), + repairs=RepairIntegrityDiagnostics( + active_modular_repairs=int(integrity["active_modular_repairs"]), + duplicate_active_repair_groups=int(integrity["duplicate_active_repair_groups"]), + legacy_repair_only_miners=int(integrity["legacy_repair_only_miners"]), + lifecycle_repair_without_active_repair=int(integrity["lifecycle_repair_without_active_repair"]), + active_repair_lifecycle_mismatches=int(integrity["active_repair_lifecycle_mismatches"]), + waiting_placement_records=int(integrity["waiting_placement_records"]), + repair_events=int(operations["repair_events"]), + repair_parts=int(operations["repair_parts"]), + ), + retention=RetentionDiagnostics( + hourly=RollupDiagnostics( + table_present=bool(operations["hourly_present"]), + row_count=int(operations["hourly_rows"]), + latest_bucket=operations.get("hourly_latest"), + ), + daily=RollupDiagnostics( + table_present=bool(operations["daily_present"]), + row_count=int(operations["daily_rows"]), + latest_bucket=operations.get("daily_latest"), + ), + latest_recorded_run_at=operations.get("retention_started_at"), + latest_recorded_run_mode=operations.get("retention_mode"), + ), + ) diff --git a/backend/tests/test_system_health.py b/backend/tests/test_system_health.py new file mode 100644 index 0000000..8221afc --- /dev/null +++ b/backend/tests/test_system_health.py @@ -0,0 +1,304 @@ +import ast +import unittest +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import psycopg +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.config import settings +from app.identity import get_current_user +from app.system_health.application import ApplicationMetadata +from app.system_health.dependencies import get_system_health_service +from app.system_health.frontend import router as frontend_router +from app.system_health.repository import SystemHealthRepository +from app.system_health.router import router as api_router +from app.system_health.service import SystemHealthService + + +APP_ROOT = Path(__file__).parents[1] / "app" +EXPECTED_HEAD = "0014_repair_parts_tracking" + + +def database_payload(revision=EXPECTED_HEAD): + return { + "migration": {"current_revision": revision}, + "operations": { + "miners": 4, "sites": 1, "rooms": 2, "location_positions": 3, + "repairs": 1, "repair_events": 2, "repair_parts": 1, + "snapshot_count": 5, + "oldest_observed_at": datetime(2026, 8, 18, tzinfo=timezone.utc), + "latest_observed_at": datetime(2026, 8, 19, tzinfo=timezone.utc), + "hourly_present": True, "hourly_rows": 2, + "hourly_latest": datetime(2026, 8, 19, tzinfo=timezone.utc), + "daily_present": True, "daily_rows": 1, + "daily_latest": datetime(2026, 8, 19, tzinfo=timezone.utc).date(), + "retention_mode": "dry_run", + "retention_started_at": datetime(2026, 8, 19, tzinfo=timezone.utc), + "largest_tables": [{"name": "miners", "total_bytes": 8192}], + }, + "integrity": { + "miners_without_asset_id": 1, "unmapped_miners": 1, + "miners_in_multiple_positions": 0, "duplicate_current_mac_groups": 0, + "unscoped_legacy_positions": 1, "expected_mac_mismatches": 1, + "occupied_positions": 2, "empty_positions": 1, + "active_modular_repairs": 1, "duplicate_active_repair_groups": 0, + "legacy_repair_only_miners": 0, + "lifecycle_repair_without_active_repair": 0, + "active_repair_lifecycle_mismatches": 0, + "waiting_placement_records": 1, + }, + } + + +class FakeRepository: + def __init__(self, payload=None, error=None): + self.payload = payload or database_payload() + self.error = error + self.calls = 0 + + def read_overview(self): + self.calls += 1 + if self.error: + raise self.error + return self.payload + + +class FakeRedis: + def __init__(self, available=True): + self.available = available + self.calls = 0 + + def ping(self): + self.calls += 1 + return self.available + + +def metadata(head=EXPECTED_HEAD): + return ApplicationMetadata( + environment="test", build_commit=None, build_version="0.1-test", + expected_migration_head=head, + ) + + +class SystemHealthServiceTests(unittest.TestCase): + def test_builds_typed_factual_overview_without_health_score(self): + repository, redis = FakeRepository(), FakeRedis() + result = SystemHealthService(repository, redis, metadata()).get_overview() + self.assertEqual(result.data.postgresql.migration.state, "current") + self.assertEqual(result.data.postgresql.telemetry.snapshot_count, 5) + self.assertEqual(result.data.postgresql.fleet.unmapped_miners, 1) + self.assertFalse(result.data.postgresql.retention.raw_retention_enabled) + self.assertEqual(result.data.postgresql.retention.hourly.population, "manual") + self.assertEqual(result.data.postgresql.retention.completeness, "not_guaranteed") + self.assertIsNone(result.data.application.build_commit) + self.assertFalse(result.meta.partial) + self.assertEqual((repository.calls, redis.calls), (1, 1)) + self.assertNotIn("score", result.model_dump_json().lower()) + + def test_migration_mismatch_is_factual(self): + result = SystemHealthService( + FakeRepository(database_payload("0013_control_board_replacement")), + FakeRedis(), metadata(), + ).get_overview() + self.assertEqual(result.data.postgresql.migration.state, "mismatch") + self.assertEqual( + result.data.postgresql.migration.current_revision, + "0013_control_board_replacement", + ) + + def test_postgresql_failure_is_sanitized_and_redis_remains_available(self): + secret = "postgresql://user:password@private-host/minerops" + result = SystemHealthService( + FakeRepository(error=RuntimeError(secret)), FakeRedis(), metadata() + ).get_overview() + self.assertTrue(result.meta.partial) + self.assertEqual(result.meta.unavailable_sections, ["postgresql"]) + self.assertEqual(result.data.postgresql.status, "unavailable") + self.assertEqual(result.data.redis.status, "available") + self.assertNotIn(secret, result.model_dump_json()) + + def test_redis_failure_preserves_database_diagnostics(self): + result = SystemHealthService( + FakeRepository(), FakeRedis(False), metadata() + ).get_overview() + self.assertEqual(result.data.postgresql.status, "available") + self.assertEqual(result.data.redis.status, "unavailable") + self.assertEqual(result.meta.unavailable_sections, ["redis"]) + + +class StubService: + def get_overview(self): + return SystemHealthService( + FakeRepository(), FakeRedis(), metadata() + ).get_overview() + + +class SystemHealthRouterTests(unittest.TestCase): + def setUp(self): + self.app = FastAPI() + self.app.include_router(api_router) + self.app.dependency_overrides[get_system_health_service] = StubService + self.client = TestClient(self.app) + + def tearDown(self): + self.client.close() + + def authenticate(self, permissions): + self.app.dependency_overrides[get_current_user] = lambda: { + "id": str(uuid4()), "role": "Custom", "permissions": list(permissions) + } + + def test_requires_authentication_and_system_health_permission(self): + self.assertEqual(self.client.get("/api/system/health-summary").status_code, 401) + self.authenticate(("dashboard.view",)) + self.assertEqual(self.client.get("/api/system/health-summary").status_code, 403) + + def test_returns_typed_no_store_envelope(self): + self.authenticate(("system_health.view",)) + response = self.client.get("/api/system/health-summary") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["cache-control"], "no-store") + payload = response.json() + self.assertEqual(payload["data"]["postgresql"]["status"], "available") + self.assertEqual(payload["meta"]["postgresql_statement_count"], 3) + self.assertIsNone(payload["error"]) + + +class CountingCursor(psycopg.Cursor): + statements = 0 + + def execute(self, query, params=None, *, prepare=None, binary=None): + type(self).statements += 1 + return super().execute( + query, params, prepare=prepare, binary=binary + ) + + +class SystemHealthRepositoryIntegrationTests(unittest.TestCase): + def setUp(self): + self.connection = psycopg.connect( + host=settings.postgres_host, port=settings.postgres_port, + dbname=settings.postgres_db, user=settings.postgres_user, + password=settings.postgres_password, cursor_factory=CountingCursor, + ) + CountingCursor.statements = 0 + self.repository = SystemHealthRepository(connection=self.connection) + + def tearDown(self): + self.connection.rollback() + self.connection.close() + + def test_real_postgresql_overview_uses_exactly_three_statements(self): + site_id, room_id, mapped_id, unmapped_id = ( + uuid4(), uuid4(), uuid4(), uuid4() + ) + suffix = uuid4().hex[:10] + with self.connection.cursor() as cur: + cur.execute("INSERT INTO sites (id, name) VALUES (%s, %s)", + (site_id, "Health Site " + suffix)) + cur.execute("INSERT INTO rooms (id, site_id, name) VALUES (%s, %s, %s)", + (room_id, site_id, "Health Room " + suffix)) + cur.execute( + "INSERT INTO miners (id, asset_tag, lifecycle_status, current_mac) " + "VALUES (%s, %s, 'active', %s), (%s, NULL, 'active', NULL)", + (mapped_id, "HEALTH-" + suffix, "02:00:00:00:00:01", unmapped_id), + ) + cur.execute( + "INSERT INTO location_positions " + "(room_id, room_name, rack_name, shelf_name, position_index, " + "miner_id, expected_mac) VALUES (%s, %s, %s, '1', 1, %s, %s), " + "(NULL, %s, %s, '1', 2, NULL, NULL)", + (room_id, "Health Room " + suffix, "Rack " + suffix, mapped_id, + "02:00:00:00:00:02", "Legacy " + suffix, "Rack " + suffix), + ) + cur.execute( + "INSERT INTO miner_status_snapshots (miner_id, observed_at, mhs_av) " + "VALUES (%s, %s, 0)", + (mapped_id, datetime(2026, 8, 19, 12, tzinfo=timezone.utc)), + ) + CountingCursor.statements = 0 + result = self.repository.read_overview() + self.assertEqual(CountingCursor.statements, 3) + self.assertGreaterEqual(result["operations"]["miners"], 2) + self.assertGreaterEqual(result["operations"]["snapshot_count"], 1) + self.assertEqual(result["migration"]["current_revision"], EXPECTED_HEAD) + self.assertGreaterEqual(result["integrity"]["unmapped_miners"], 1) + self.assertGreaterEqual(result["integrity"]["unscoped_legacy_positions"], 1) + self.assertGreaterEqual(result["integrity"]["expected_mac_mismatches"], 1) + self.assertTrue(result["operations"]["hourly_present"]) + self.assertTrue(result["operations"]["daily_present"]) + + def test_existing_role_matrix_preserves_read_only_grant(self): + with self.connection.cursor() as cur: + cur.execute( + "SELECT roles.name FROM role_permissions " + "JOIN roles ON roles.id = role_permissions.role_id " + "JOIN permissions ON permissions.id = role_permissions.permission_id " + "WHERE permissions.key = 'system_health.view' ORDER BY roles.name" + ) + roles = [row[0] for row in cur.fetchall()] + self.assertEqual(roles, ["Admin", "Manager", "Read Only"]) + + +class SystemHealthFrontendAndArchitectureTests(unittest.TestCase): + def test_page_is_semantic_safe_and_one_request(self): + app = FastAPI() + app.include_router(frontend_router) + app.dependency_overrides[get_current_user] = lambda: { + "id": str(uuid4()), "display_name": "Health ", + "role": "Manager", "permissions": ["system_health.view"], + } + client = TestClient(app) + response = client.get("/system-health") + self.assertEqual(response.status_code, 200) + self.assertIn("System Health | MinerOps", response.text) + self.assertNotIn("Health ", response.text) + self.assertIn("aria-live=\"polite\"", response.text) + client.close() + script = (APP_ROOT / "frontend/static/system-health.js").read_text() + self.assertEqual(script.count('fetch("/api/system/health-summary"'), 1) + self.assertNotIn("setInterval", script) + self.assertNotIn("innerHTML", script) + self.assertIn("textContent", script) + + def test_routes_are_unique_and_legacy_source_diagnostics_are_removed(self): + source = (APP_ROOT / "main.py").read_text() + self.assertNotIn("source_lines", source) + self.assertNotIn("recent_patch_markers", source) + from app.main import app + page = [r for r in app.routes if getattr(r, "path", None) == "/system-health"] + api = [r for r in app.routes if getattr(r, "path", None) == "/api/system/health-summary"] + self.assertEqual(len(page), 1) + self.assertEqual(len(api), 1) + self.assertEqual(page[0].methods, {"GET"}) + self.assertEqual(api[0].methods, {"GET"}) + + def test_layers_and_security_boundaries(self): + router_source = (APP_ROOT / "system_health/router.py").read_text() + service_source = (APP_ROOT / "system_health/service.py").read_text() + repository_source = (APP_ROOT / "system_health/repository.py").read_text() + ast.parse(router_source) + ast.parse(service_source) + self.assertNotIn("SELECT ", router_source) + self.assertNotIn("SELECT ", service_source) + self.assertNotIn("fastapi", repository_source.lower()) + self.assertNotIn("fastapi", service_source.lower()) + static = "\n".join( + (APP_ROOT / path).read_text() + for path in ( + "frontend/static/system-health.js", + "frontend/static/system-health.css", + ) + ).lower() + for forbidden in ( + "postgres_password", "redis_url", "jwt_secret", "connection string", + "stack trace", "password hash", "api key", + ): + self.assertNotIn(forbidden, static) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-system-health.sh b/scripts/test-system-health.sh new file mode 100755 index 0000000..c8e4d3e --- /dev/null +++ b/scripts/test-system-health.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +readonly source_files=( + backend/app/config.py + backend/app/frontend/static/system-health.css + backend/app/frontend/static/system-health.js + backend/app/frontend/system_health_view.py + backend/app/frontend/views.py + backend/app/main.py + backend/app/system_health/__init__.py + backend/app/system_health/adapters.py + backend/app/system_health/application.py + backend/app/system_health/dependencies.py + backend/app/system_health/frontend.py + backend/app/system_health/models.py + backend/app/system_health/repository.py + backend/app/system_health/router.py + backend/app/system_health/service.py + backend/tests/__init__.py + backend/tests/test_frontend_shell.py + backend/tests/test_system_health.py +) + +for file in "${source_files[@]}"; do + [[ -r "${file}" ]] || { echo "Unreadable System Health input: ${file}" >&2; exit 1; } + [[ "$(stat -c '%a' "${file}")" == "644" ]] || { + echo "System Health source/test mode must be 0644: ${file}" >&2; exit 1; + } +done +[[ "$(stat -c '%a' scripts/test-system-health.sh)" == "755" ]] || { + echo "scripts/test-system-health.sh must have mode 0755." >&2; exit 1; +} +echo "System Health file-mode preflight passed." + +python3 -m py_compile backend/app/config.py backend/app/frontend/system_health_view.py backend/app/frontend/views.py backend/app/main.py backend/app/system_health/*.py backend/tests/__init__.py backend/tests/test_frontend_shell.py backend/tests/test_system_health.py +node --check backend/app/frontend/static/system-health.js +bash -n scripts/test-system-health.sh + +[[ -f .env.dev ]] || { echo "Missing .env.dev." >&2; exit 1; } +project_name="${MINEROPS_SYSTEM_HEALTH_TEST_PROJECT:-minerops-system-health-$$}" +compose=(docker compose --project-name "${project_name}" --env-file .env.dev -f compose.dev.yaml) +cleanup() { + status=$? + if (( status != 0 )); then + "${compose[@]}" logs --no-color --tail 100 postgres redis || true + fi + "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true + exit "${status}" +} +trap cleanup EXIT + +"${compose[@]}" up --detach postgres redis +for attempt in $(seq 1 60); do + if "${compose[@]}" exec -T postgres sh -ec \ + 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null 2>&1 \ + && "${compose[@]}" exec -T redis redis-cli ping 2>/dev/null | grep -qx PONG; then + break + fi + (( attempt < 60 )) || { echo "Dependencies did not become ready." >&2; exit 1; } + sleep 1 +done +"${compose[@]}" build backend +"${compose[@]}" run --rm --no-deps backend alembic upgrade head +"${compose[@]}" run --rm --no-deps \ + -v "${repo_root}/backend/tests:/app/tests:ro" \ + backend python -m unittest -v tests.test_system_health tests.test_frontend_shell