137 lines
6.4 KiB
Python
137 lines
6.4 KiB
Python
"""Composition and failure semantics for System Health diagnostics."""
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from app.system_health.adapters import RedisHealthAdapter
|
|
from app.system_health.application import APPLICATION_METADATA, ApplicationMetadata
|
|
from app.system_health.models import (
|
|
ApplicationDiagnostics, FleetIntegrityDiagnostics, LocationIntegrityDiagnostics,
|
|
MigrationDiagnostics, PostgreSQLDiagnostics, RedisDiagnostics,
|
|
RepairIntegrityDiagnostics, RetentionDiagnostics, RollupDiagnostics,
|
|
SystemHealthOverviewData, SystemHealthOverviewMeta,
|
|
SystemHealthOverviewResponse, TableCount, TableSize, TelemetryDiagnostics,
|
|
)
|
|
from app.system_health.repository import SystemHealthRepository
|
|
|
|
|
|
class SystemHealthService:
|
|
def __init__(self, repository: SystemHealthRepository,
|
|
redis_adapter: RedisHealthAdapter,
|
|
metadata: ApplicationMetadata = APPLICATION_METADATA):
|
|
self.repository = repository
|
|
self.redis_adapter = redis_adapter
|
|
self.metadata = metadata
|
|
|
|
def get_overview(self) -> SystemHealthOverviewResponse:
|
|
generated_at = datetime.now(timezone.utc)
|
|
unavailable = []
|
|
try:
|
|
postgresql = self._postgresql_diagnostics(
|
|
self.repository.read_overview()
|
|
)
|
|
except Exception:
|
|
unavailable.append("postgresql")
|
|
postgresql = PostgreSQLDiagnostics(
|
|
status="unavailable",
|
|
migration=MigrationDiagnostics(
|
|
state="unavailable",
|
|
expected_revision=self.metadata.expected_migration_head,
|
|
),
|
|
)
|
|
redis_available = self.redis_adapter.ping()
|
|
if not redis_available:
|
|
unavailable.append("redis")
|
|
return SystemHealthOverviewResponse(
|
|
data=SystemHealthOverviewData(
|
|
application=ApplicationDiagnostics(
|
|
environment=self.metadata.environment,
|
|
build_commit=self.metadata.build_commit,
|
|
build_version=self.metadata.build_version,
|
|
server_time=generated_at,
|
|
expected_migration_head=self.metadata.expected_migration_head,
|
|
),
|
|
postgresql=postgresql,
|
|
redis=RedisDiagnostics(
|
|
status="available" if redis_available else "unavailable"
|
|
),
|
|
),
|
|
meta=SystemHealthOverviewMeta(
|
|
generated_at=generated_at,
|
|
partial=bool(unavailable),
|
|
unavailable_sections=unavailable,
|
|
),
|
|
)
|
|
|
|
def _postgresql_diagnostics(self, payload: dict) -> PostgreSQLDiagnostics:
|
|
migration = payload["migration"]
|
|
operations = payload["operations"]
|
|
integrity = payload["integrity"]
|
|
current = migration.get("current_revision")
|
|
expected = self.metadata.expected_migration_head
|
|
state = (
|
|
"unavailable" if not current or not expected
|
|
else "current" if current == expected else "mismatch"
|
|
)
|
|
counts = [
|
|
TableCount(name=name, row_count=int(operations[name]))
|
|
for name in (
|
|
"miners", "sites", "rooms", "location_positions", "repairs",
|
|
"repair_events", "repair_parts", "snapshot_count",
|
|
)
|
|
]
|
|
return PostgreSQLDiagnostics(
|
|
status="available",
|
|
migration=MigrationDiagnostics(
|
|
state=state, current_revision=current, expected_revision=expected
|
|
),
|
|
table_counts=counts,
|
|
largest_tables=[
|
|
TableSize(name=row["name"], total_bytes=int(row["total_bytes"]))
|
|
for row in operations.get("largest_tables") or []
|
|
],
|
|
telemetry=TelemetryDiagnostics(
|
|
snapshot_count=int(operations["snapshot_count"]),
|
|
oldest_observed_at=operations.get("oldest_observed_at"),
|
|
latest_observed_at=operations.get("latest_observed_at"),
|
|
),
|
|
fleet=FleetIntegrityDiagnostics(
|
|
total_miners=int(operations["miners"]),
|
|
miners_without_asset_id=int(integrity["miners_without_asset_id"]),
|
|
unmapped_miners=int(integrity["unmapped_miners"]),
|
|
miners_in_multiple_positions=int(integrity["miners_in_multiple_positions"]),
|
|
duplicate_current_mac_groups=int(integrity["duplicate_current_mac_groups"]),
|
|
),
|
|
locations=LocationIntegrityDiagnostics(
|
|
sites=int(operations["sites"]), rooms=int(operations["rooms"]),
|
|
positions=int(operations["location_positions"]),
|
|
occupied_positions=int(integrity["occupied_positions"]),
|
|
empty_positions=int(integrity["empty_positions"]),
|
|
unscoped_legacy_positions=int(integrity["unscoped_legacy_positions"]),
|
|
expected_mac_mismatches=int(integrity["expected_mac_mismatches"]),
|
|
),
|
|
repairs=RepairIntegrityDiagnostics(
|
|
active_modular_repairs=int(integrity["active_modular_repairs"]),
|
|
duplicate_active_repair_groups=int(integrity["duplicate_active_repair_groups"]),
|
|
legacy_repair_only_miners=int(integrity["legacy_repair_only_miners"]),
|
|
lifecycle_repair_without_active_repair=int(integrity["lifecycle_repair_without_active_repair"]),
|
|
active_repair_lifecycle_mismatches=int(integrity["active_repair_lifecycle_mismatches"]),
|
|
waiting_placement_records=int(integrity["waiting_placement_records"]),
|
|
repair_events=int(operations["repair_events"]),
|
|
repair_parts=int(operations["repair_parts"]),
|
|
),
|
|
retention=RetentionDiagnostics(
|
|
hourly=RollupDiagnostics(
|
|
table_present=bool(operations["hourly_present"]),
|
|
row_count=int(operations["hourly_rows"]),
|
|
latest_bucket=operations.get("hourly_latest"),
|
|
),
|
|
daily=RollupDiagnostics(
|
|
table_present=bool(operations["daily_present"]),
|
|
row_count=int(operations["daily_rows"]),
|
|
latest_bucket=operations.get("daily_latest"),
|
|
),
|
|
latest_recorded_run_at=operations.get("retention_started_at"),
|
|
latest_recorded_run_mode=operations.get("retention_mode"),
|
|
),
|
|
)
|