feat: add system health diagnostics

backend/app/system_health/__init__.py

@@ -0,0 +1 @@
+"""System Health diagnostics domain."""

backend/app/system_health/adapters.py

@@ -0,0 +1,29 @@
+"""External service adapters for System Health."""
+
+from collections.abc import Callable
+
+import redis
+
+from app.config import settings
+
+
+RedisFactory = Callable[[], redis.Redis]
+
+
+def default_redis_factory() -> redis.Redis:
+    return redis.Redis.from_url(
+        settings.redis_url,
+        socket_connect_timeout=2,
+        socket_timeout=2,
+    )
+
+
+class RedisHealthAdapter:
+    def __init__(self, factory: RedisFactory | None = None):
+        self._factory = factory or default_redis_factory
+
+    def ping(self) -> bool:
+        try:
+            return bool(self._factory().ping())
+        except Exception:
+            return False

backend/app/system_health/application.py

@@ -0,0 +1,42 @@
+"""Startup-cached, non-secret application metadata."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+
+from app.config import settings
+
+
+def _clean_build_value(value: str | None) -> str | None:
+    if value is None:
+        return None
+    cleaned = value.strip()
+    return cleaned[:128] if cleaned else None
+
+
+def _expected_migration_head() -> str | None:
+    try:
+        backend_root = Path(__file__).resolve().parents[2]
+        config = Config(str(backend_root / "alembic.ini"))
+        config.set_main_option("script_location", str(backend_root / "alembic"))
+        return ScriptDirectory.from_config(config).get_current_head()
+    except Exception:
+        return None
+
+
+@dataclass(frozen=True)
+class ApplicationMetadata:
+    environment: str
+    build_commit: str | None
+    build_version: str | None
+    expected_migration_head: str | None
+
+
+APPLICATION_METADATA = ApplicationMetadata(
+    environment=settings.app_env,
+    build_commit=_clean_build_value(settings.minerops_build_commit),
+    build_version=_clean_build_value(settings.minerops_build_version),
+    expected_migration_head=_expected_migration_head(),
+)

backend/app/system_health/dependencies.py

@@ -0,0 +1,9 @@
+"""Dependency construction for System Health."""
+
+from app.system_health.adapters import RedisHealthAdapter
+from app.system_health.repository import SystemHealthRepository
+from app.system_health.service import SystemHealthService
+
+
+def get_system_health_service() -> SystemHealthService:
+    return SystemHealthService(SystemHealthRepository(), RedisHealthAdapter())

backend/app/system_health/frontend.py

@@ -0,0 +1,35 @@
+"""Authenticated frontend routes for System Health."""
+
+from pathlib import Path
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import HTMLResponse, Response
+
+from app.frontend.views import render_app_page
+from app.identity import require_permission, user_has_permission
+
+
+router = APIRouter(tags=["frontend"])
+STATIC_ROOT = Path(__file__).parents[1] / "frontend" / "static"
+SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))]
+
+
+@router.get("/assets/minerops-system-health.css", include_in_schema=False)
+def system_health_styles() -> Response:
+    return Response((STATIC_ROOT / "system-health.css").read_text(encoding="utf-8"),
+                    media_type="text/css", headers={"Cache-Control": "no-store"})
+
+
+@router.get("/assets/minerops-system-health.js", include_in_schema=False)
+def system_health_script() -> Response:
+    return Response((STATIC_ROOT / "system-health.js").read_text(encoding="utf-8"),
+                    media_type="text/javascript", headers={"Cache-Control": "no-store"})
+
+
+@router.get("/system-health", response_class=HTMLResponse, include_in_schema=False)
+def system_health_page(current_user: SystemHealthUser) -> str:
+    return render_app_page(
+        "system_health", current_user,
+        lambda permission: user_has_permission(current_user, permission),
+    )

backend/app/system_health/models.py

@@ -0,0 +1,124 @@
+"""Typed contracts for the System Health overview."""
+
+from datetime import date, datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+Availability = Literal["available", "unavailable"]
+MigrationState = Literal["current", "mismatch", "unavailable"]
+
+
+class ApplicationDiagnostics(BaseModel):
+    status: Literal["available"] = "available"
+    environment: str
+    build_commit: str | None = None
+    build_version: str | None = None
+    server_time: datetime
+    expected_migration_head: str | None = None
+
+
+class MigrationDiagnostics(BaseModel):
+    state: MigrationState
+    current_revision: str | None = None
+    expected_revision: str | None = None
+
+
+class TableCount(BaseModel):
+    name: str
+    row_count: int
+
+
+class TableSize(BaseModel):
+    name: str
+    total_bytes: int
+
+
+class TelemetryDiagnostics(BaseModel):
+    snapshot_count: int
+    oldest_observed_at: datetime | None = None
+    latest_observed_at: datetime | None = None
+
+
+class FleetIntegrityDiagnostics(BaseModel):
+    total_miners: int
+    miners_without_asset_id: int
+    unmapped_miners: int
+    miners_in_multiple_positions: int
+    duplicate_current_mac_groups: int
+
+
+class LocationIntegrityDiagnostics(BaseModel):
+    sites: int
+    rooms: int
+    positions: int
+    occupied_positions: int
+    empty_positions: int
+    unscoped_legacy_positions: int
+    expected_mac_mismatches: int
+
+
+class RepairIntegrityDiagnostics(BaseModel):
+    active_modular_repairs: int
+    duplicate_active_repair_groups: int
+    legacy_repair_only_miners: int
+    lifecycle_repair_without_active_repair: int
+    active_repair_lifecycle_mismatches: int
+    waiting_placement_records: int
+    repair_events: int
+    repair_parts: int
+
+
+class RollupDiagnostics(BaseModel):
+    table_present: bool
+    row_count: int | None = None
+    latest_bucket: datetime | date | None = None
+    population: Literal["manual"] = "manual"
+
+
+class RetentionDiagnostics(BaseModel):
+    raw_retention_enabled: Literal[False] = False
+    raw_retention_mode: Literal["not_enabled"] = "not_enabled"
+    hourly: RollupDiagnostics
+    daily: RollupDiagnostics
+    latest_recorded_run_at: datetime | None = None
+    latest_recorded_run_mode: str | None = None
+    completeness: Literal["not_guaranteed"] = "not_guaranteed"
+
+
+class PostgreSQLDiagnostics(BaseModel):
+    status: Availability
+    migration: MigrationDiagnostics
+    table_counts: list[TableCount] = Field(default_factory=list)
+    largest_tables: list[TableSize] = Field(default_factory=list)
+    telemetry: TelemetryDiagnostics | None = None
+    fleet: FleetIntegrityDiagnostics | None = None
+    locations: LocationIntegrityDiagnostics | None = None
+    repairs: RepairIntegrityDiagnostics | None = None
+    retention: RetentionDiagnostics | None = None
+
+
+class RedisDiagnostics(BaseModel):
+    status: Availability
+
+
+class SystemHealthOverviewData(BaseModel):
+    application: ApplicationDiagnostics
+    postgresql: PostgreSQLDiagnostics
+    redis: RedisDiagnostics
+
+
+class SystemHealthOverviewMeta(BaseModel):
+    generated_at: datetime
+    partial: bool
+    unavailable_sections: list[str] = Field(default_factory=list)
+    frontend_request_count: Literal[1] = 1
+    postgresql_statement_count: Literal[3] = 3
+    redis_call_count: Literal[1] = 1
+
+
+class SystemHealthOverviewResponse(BaseModel):
+    data: SystemHealthOverviewData
+    meta: SystemHealthOverviewMeta
+    error: None = None

backend/app/system_health/repository.py

@@ -0,0 +1,179 @@
+"""PostgreSQL diagnostics for System Health."""
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from typing import Any
+
+import psycopg
+from psycopg.rows import dict_row
+
+from app.config import settings
+
+
+ConnectionFactory = Callable[[], psycopg.Connection]
+
+
+def default_connection_factory() -> psycopg.Connection:
+    return psycopg.connect(
+        host=settings.postgres_host,
+        port=settings.postgres_port,
+        dbname=settings.postgres_db,
+        user=settings.postgres_user,
+        password=settings.postgres_password,
+        connect_timeout=3,
+    )
+
+
+MIGRATION_SQL = """
+SELECT clock_timestamp() AS database_time,
+       (SELECT version_num FROM alembic_version LIMIT 1) AS current_revision
+"""
+
+OPERATIONS_SQL = """
+WITH telemetry AS (
+    SELECT count(*)::bigint AS snapshot_count,
+           min(observed_at) AS oldest_observed_at,
+           max(observed_at) AS latest_observed_at
+    FROM miner_status_snapshots
+),
+hourly AS (
+    SELECT count(*)::bigint AS row_count, max(bucket_at) AS latest_bucket
+    FROM miner_status_hourly_rollups
+),
+daily AS (
+    SELECT count(*)::bigint AS row_count, max(bucket_date) AS latest_bucket
+    FROM miner_status_daily_rollups
+),
+latest_retention AS (
+    SELECT mode, started_at FROM minerops_retention_runs
+    ORDER BY started_at DESC LIMIT 1
+),
+largest AS (
+    SELECT coalesce(jsonb_agg(to_jsonb(sized) ORDER BY sized.total_bytes DESC), '[]'::jsonb) AS rows
+    FROM (
+        SELECT relname AS name, pg_total_relation_size(relid)::bigint AS total_bytes
+        FROM pg_catalog.pg_statio_user_tables
+        WHERE relname = ANY(%s)
+        ORDER BY pg_total_relation_size(relid) DESC, relname
+        LIMIT 8
+    ) AS sized
+)
+SELECT
+    (SELECT count(*) FROM miners)::bigint AS miners,
+    (SELECT count(*) FROM sites)::bigint AS sites,
+    (SELECT count(*) FROM rooms)::bigint AS rooms,
+    (SELECT count(*) FROM location_positions)::bigint AS location_positions,
+    (SELECT count(*) FROM repairs)::bigint AS repairs,
+    (SELECT count(*) FROM repair_events)::bigint AS repair_events,
+    (SELECT count(*) FROM repair_parts)::bigint AS repair_parts,
+    telemetry.snapshot_count, telemetry.oldest_observed_at,
+    telemetry.latest_observed_at,
+    to_regclass('public.miner_status_hourly_rollups') IS NOT NULL AS hourly_present,
+    hourly.row_count AS hourly_rows, hourly.latest_bucket AS hourly_latest,
+    to_regclass('public.miner_status_daily_rollups') IS NOT NULL AS daily_present,
+    daily.row_count AS daily_rows, daily.latest_bucket AS daily_latest,
+    latest_retention.mode AS retention_mode,
+    latest_retention.started_at AS retention_started_at,
+    largest.rows AS largest_tables
+FROM telemetry CROSS JOIN hourly CROSS JOIN daily CROSS JOIN largest
+LEFT JOIN latest_retention ON true
+"""
+
+INTEGRITY_SQL = """
+WITH position_counts AS (
+    SELECT miner_id, count(*) AS position_count
+    FROM location_positions WHERE miner_id IS NOT NULL GROUP BY miner_id
+),
+active_repairs AS (
+    SELECT id, miner_id FROM repairs
+    WHERE status IN ('OPEN', 'IN_PROGRESS', 'WAITING_FOR_PARTS', 'READY_FOR_TEST')
+),
+active_repair_groups AS (
+    SELECT miner_id FROM active_repairs GROUP BY miner_id HAVING count(*) > 1
+),
+normalized_macs AS (
+    SELECT lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g')) AS mac
+    FROM miners WHERE current_mac IS NOT NULL AND trim(current_mac) <> ''
+    GROUP BY lower(regexp_replace(trim(current_mac), '[^0-9A-Fa-f]', '', 'g'))
+    HAVING count(*) > 1
+)
+SELECT
+    (SELECT count(*) FROM miners WHERE asset_tag IS NULL OR trim(asset_tag) = '')::bigint
+        AS miners_without_asset_id,
+    (SELECT count(*) FROM miners m WHERE NOT EXISTS (
+        SELECT 1 FROM location_positions p WHERE p.miner_id = m.id
+    ))::bigint AS unmapped_miners,
+    (SELECT count(*) FROM position_counts WHERE position_count > 1)::bigint
+        AS miners_in_multiple_positions,
+    (SELECT count(*) FROM normalized_macs)::bigint AS duplicate_current_mac_groups,
+    (SELECT count(*) FROM location_positions WHERE room_id IS NULL)::bigint
+        AS unscoped_legacy_positions,
+    (SELECT count(*) FROM location_positions p JOIN miners m ON m.id = p.miner_id
+     WHERE p.expected_mac IS NOT NULL AND trim(p.expected_mac) <> ''
+       AND m.current_mac IS NOT NULL AND trim(m.current_mac) <> ''
+       AND lower(regexp_replace(trim(p.expected_mac), '[^0-9A-Fa-f]', '', 'g'))
+           <> lower(regexp_replace(trim(m.current_mac), '[^0-9A-Fa-f]', '', 'g'))
+    )::bigint AS expected_mac_mismatches,
+    (SELECT count(*) FROM location_positions WHERE miner_id IS NOT NULL)::bigint
+        AS occupied_positions,
+    (SELECT count(*) FROM location_positions WHERE miner_id IS NULL)::bigint
+        AS empty_positions,
+    (SELECT count(*) FROM active_repairs)::bigint AS active_modular_repairs,
+    (SELECT count(*) FROM active_repair_groups)::bigint AS duplicate_active_repair_groups,
+    (SELECT count(*) FROM miner_location_states state
+     WHERE state.asset_state = 'repair' AND NOT EXISTS (
+         SELECT 1 FROM active_repairs repair WHERE repair.miner_id = state.miner_id
+     ))::bigint AS legacy_repair_only_miners,
+    (SELECT count(*) FROM miners m WHERE m.lifecycle_status = 'in_repair'
+     AND NOT EXISTS (SELECT 1 FROM active_repairs r WHERE r.miner_id = m.id))::bigint
+        AS lifecycle_repair_without_active_repair,
+    (SELECT count(*) FROM active_repairs r JOIN miners m ON m.id = r.miner_id
+     WHERE m.lifecycle_status <> 'in_repair')::bigint
+        AS active_repair_lifecycle_mismatches,
+    (SELECT count(*) FROM miner_location_states WHERE asset_state = 'waiting_placement')::bigint
+        AS waiting_placement_records
+"""
+
+
+RELEVANT_TABLES = [
+    "miner_status_snapshots", "repair_events", "repair_parts", "repairs",
+    "location_positions", "miners", "miner_status_hourly_rollups",
+    "miner_status_daily_rollups",
+]
+
+
+class SystemHealthRepository:
+    def __init__(
+        self,
+        connection_factory: ConnectionFactory | None = None,
+        *,
+        connection: psycopg.Connection | None = None,
+    ):
+        self._connection_factory = connection_factory or default_connection_factory
+        self._connection = connection
+
+    @contextmanager
+    def _connection_scope(self) -> Iterator[psycopg.Connection]:
+        if self._connection is not None:
+            yield self._connection
+            return
+        connection = self._connection_factory()
+        try:
+            yield connection
+        finally:
+            connection.close()
+
+    def read_overview(self) -> dict[str, Any]:
+        with self._connection_scope() as connection:
+            with connection.cursor(row_factory=dict_row) as cur:
+                cur.execute(MIGRATION_SQL)
+                migration = dict(cur.fetchone())
+                cur.execute(OPERATIONS_SQL, (RELEVANT_TABLES,))
+                operations = dict(cur.fetchone())
+                cur.execute(INTEGRITY_SQL)
+                integrity = dict(cur.fetchone())
+        return {
+            "migration": migration,
+            "operations": operations,
+            "integrity": integrity,
+        }

backend/app/system_health/router.py

@@ -0,0 +1,27 @@
+"""HTTP boundary for the System Health overview."""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Response
+
+from app.identity import require_permission
+from app.system_health.dependencies import get_system_health_service
+from app.system_health.models import SystemHealthOverviewResponse
+from app.system_health.service import SystemHealthService
+
+
+router = APIRouter(prefix="/api/system", tags=["system-health"])
+SystemHealthUser = Annotated[dict, Depends(require_permission("system_health.view"))]
+SystemHealthServiceDependency = Annotated[
+    SystemHealthService, Depends(get_system_health_service)
+]
+
+
+@router.get("/health-summary", response_model=SystemHealthOverviewResponse)
+def get_system_health_overview(
+    current_user: SystemHealthUser,
+    service: SystemHealthServiceDependency,
+    response: Response,
+) -> SystemHealthOverviewResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.get_overview()

backend/app/system_health/service.py

@@ -0,0 +1,136 @@
+"""Composition and failure semantics for System Health diagnostics."""
+
+from datetime import datetime, timezone
+
+from app.system_health.adapters import RedisHealthAdapter
+from app.system_health.application import APPLICATION_METADATA, ApplicationMetadata
+from app.system_health.models import (
+    ApplicationDiagnostics, FleetIntegrityDiagnostics, LocationIntegrityDiagnostics,
+    MigrationDiagnostics, PostgreSQLDiagnostics, RedisDiagnostics,
+    RepairIntegrityDiagnostics, RetentionDiagnostics, RollupDiagnostics,
+    SystemHealthOverviewData, SystemHealthOverviewMeta,
+    SystemHealthOverviewResponse, TableCount, TableSize, TelemetryDiagnostics,
+)
+from app.system_health.repository import SystemHealthRepository
+
+
+class SystemHealthService:
+    def __init__(self, repository: SystemHealthRepository,
+                 redis_adapter: RedisHealthAdapter,
+                 metadata: ApplicationMetadata = APPLICATION_METADATA):
+        self.repository = repository
+        self.redis_adapter = redis_adapter
+        self.metadata = metadata
+
+    def get_overview(self) -> SystemHealthOverviewResponse:
+        generated_at = datetime.now(timezone.utc)
+        unavailable = []
+        try:
+            postgresql = self._postgresql_diagnostics(
+                self.repository.read_overview()
+            )
+        except Exception:
+            unavailable.append("postgresql")
+            postgresql = PostgreSQLDiagnostics(
+                status="unavailable",
+                migration=MigrationDiagnostics(
+                    state="unavailable",
+                    expected_revision=self.metadata.expected_migration_head,
+                ),
+            )
+        redis_available = self.redis_adapter.ping()
+        if not redis_available:
+            unavailable.append("redis")
+        return SystemHealthOverviewResponse(
+            data=SystemHealthOverviewData(
+                application=ApplicationDiagnostics(
+                    environment=self.metadata.environment,
+                    build_commit=self.metadata.build_commit,
+                    build_version=self.metadata.build_version,
+                    server_time=generated_at,
+                    expected_migration_head=self.metadata.expected_migration_head,
+                ),
+                postgresql=postgresql,
+                redis=RedisDiagnostics(
+                    status="available" if redis_available else "unavailable"
+                ),
+            ),
+            meta=SystemHealthOverviewMeta(
+                generated_at=generated_at,
+                partial=bool(unavailable),
+                unavailable_sections=unavailable,
+            ),
+        )
+
+    def _postgresql_diagnostics(self, payload: dict) -> PostgreSQLDiagnostics:
+        migration = payload["migration"]
+        operations = payload["operations"]
+        integrity = payload["integrity"]
+        current = migration.get("current_revision")
+        expected = self.metadata.expected_migration_head
+        state = (
+            "unavailable" if not current or not expected
+            else "current" if current == expected else "mismatch"
+        )
+        counts = [
+            TableCount(name=name, row_count=int(operations[name]))
+            for name in (
+                "miners", "sites", "rooms", "location_positions", "repairs",
+                "repair_events", "repair_parts", "snapshot_count",
+            )
+        ]
+        return PostgreSQLDiagnostics(
+            status="available",
+            migration=MigrationDiagnostics(
+                state=state, current_revision=current, expected_revision=expected
+            ),
+            table_counts=counts,
+            largest_tables=[
+                TableSize(name=row["name"], total_bytes=int(row["total_bytes"]))
+                for row in operations.get("largest_tables") or []
+            ],
+            telemetry=TelemetryDiagnostics(
+                snapshot_count=int(operations["snapshot_count"]),
+                oldest_observed_at=operations.get("oldest_observed_at"),
+                latest_observed_at=operations.get("latest_observed_at"),
+            ),
+            fleet=FleetIntegrityDiagnostics(
+                total_miners=int(operations["miners"]),
+                miners_without_asset_id=int(integrity["miners_without_asset_id"]),
+                unmapped_miners=int(integrity["unmapped_miners"]),
+                miners_in_multiple_positions=int(integrity["miners_in_multiple_positions"]),
+                duplicate_current_mac_groups=int(integrity["duplicate_current_mac_groups"]),
+            ),
+            locations=LocationIntegrityDiagnostics(
+                sites=int(operations["sites"]), rooms=int(operations["rooms"]),
+                positions=int(operations["location_positions"]),
+                occupied_positions=int(integrity["occupied_positions"]),
+                empty_positions=int(integrity["empty_positions"]),
+                unscoped_legacy_positions=int(integrity["unscoped_legacy_positions"]),
+                expected_mac_mismatches=int(integrity["expected_mac_mismatches"]),
+            ),
+            repairs=RepairIntegrityDiagnostics(
+                active_modular_repairs=int(integrity["active_modular_repairs"]),
+                duplicate_active_repair_groups=int(integrity["duplicate_active_repair_groups"]),
+                legacy_repair_only_miners=int(integrity["legacy_repair_only_miners"]),
+                lifecycle_repair_without_active_repair=int(integrity["lifecycle_repair_without_active_repair"]),
+                active_repair_lifecycle_mismatches=int(integrity["active_repair_lifecycle_mismatches"]),
+                waiting_placement_records=int(integrity["waiting_placement_records"]),
+                repair_events=int(operations["repair_events"]),
+                repair_parts=int(operations["repair_parts"]),
+            ),
+            retention=RetentionDiagnostics(
+                hourly=RollupDiagnostics(
+                    table_present=bool(operations["hourly_present"]),
+                    row_count=int(operations["hourly_rows"]),
+                    latest_bucket=operations.get("hourly_latest"),
+                ),
+                daily=RollupDiagnostics(
+                    table_present=bool(operations["daily_present"]),
+                    row_count=int(operations["daily_rows"]),
+                    latest_bucket=operations.get("daily_latest"),
+                ),
+                latest_recorded_run_at=operations.get("retention_started_at"),
+                latest_recorded_run_mode=operations.get("retention_mode"),
+            ),
+        )