feat: add monitoring trends
This commit is contained in:
lussierj committed
1 parent
19fc29c72c
commit
4d360fac42
18 files changed
+1683
No files matched your search
@@ -0,0 +1,448 @@
|
||||
import ast
|
||||
import time
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import psycopg
|
||||
|
||||
from app.config import settings
|
||||
from app.identity import get_current_user, protected_page, required_page_permission
|
||||
from app.monitoring.dependencies import get_monitoring_service
|
||||
from app.monitoring.frontend import router as monitoring_frontend_router
|
||||
from app.monitoring.models import MonitoringTrendQuery
|
||||
from app.monitoring.repository import MonitoringRepository, TREND_SQL
|
||||
from app.monitoring.router import router as monitoring_router
|
||||
from app.monitoring.service import MonitoringService
|
||||
|
||||
|
||||
NOW = datetime(2026, 8, 17, 12, 2, tzinfo=timezone.utc)
|
||||
APP_ROOT = Path(__file__).parents[1] / "app"
|
||||
STATIC_ROOT = APP_ROOT / "frontend" / "static"
|
||||
|
||||
|
||||
def trend_rows(count=72, *, scope_exists=True, label="Fleet"):
|
||||
start = datetime(2026, 8, 17, 6, 5, tzinfo=timezone.utc)
|
||||
rows = []
|
||||
for index in range(count):
|
||||
bucket_start = start + timedelta(minutes=5 * index)
|
||||
rows.append({
|
||||
"scope_exists": scope_exists,
|
||||
"scope_label": label,
|
||||
"bucket_start": bucket_start,
|
||||
"bucket_end": bucket_start + timedelta(minutes=5),
|
||||
"hashrate_ths": 0 if index == 0 else None,
|
||||
"power_w": 0 if index == 0 else None,
|
||||
"observation_count": 1 if index < 2 else 0,
|
||||
"hashrate_sample_count": 1 if index == 0 else 0,
|
||||
"power_sample_count": 1 if index == 0 else 0,
|
||||
"observed_miner_count": 1 if index < 2 else 0,
|
||||
"hashrate_contributing_miner_count": 1 if index == 0 else 0,
|
||||
"power_contributing_miner_count": 1 if index == 0 else 0,
|
||||
"scoped_miner_count": 2,
|
||||
"latest_observed_at": start + timedelta(minutes=1),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
class FakeMonitoringRepository:
|
||||
def __init__(self, rows=None):
|
||||
self.rows = rows or trend_rows()
|
||||
self.trend_calls = []
|
||||
self.option_calls = 0
|
||||
|
||||
def read_trends(self, **kwargs):
|
||||
self.trend_calls.append(kwargs)
|
||||
return self.rows
|
||||
|
||||
def list_location_scope_options(self):
|
||||
self.option_calls += 1
|
||||
site_id, room_id = uuid4(), uuid4()
|
||||
return [
|
||||
{"option_type": "site", "id": site_id, "site_id": None,
|
||||
"name": '<script>alert("site")</script>'},
|
||||
{"option_type": "room", "id": room_id, "site_id": site_id,
|
||||
"name": "Room 1"},
|
||||
]
|
||||
|
||||
|
||||
class MonitoringServiceTests(unittest.TestCase):
|
||||
def test_fixed_ranges_emit_expected_bounded_points(self):
|
||||
expected = {"6h": (72, 300), "24h": (96, 900), "7d": (168, 3600)}
|
||||
for range_name, (count, seconds) in expected.items():
|
||||
with self.subTest(range_name=range_name):
|
||||
rows = trend_rows(count)
|
||||
repository = FakeMonitoringRepository(rows)
|
||||
response = MonitoringService(repository).get_trends(
|
||||
MonitoringTrendQuery(scope="fleet", range=range_name),
|
||||
include_location_scopes=False,
|
||||
generated_at=NOW,
|
||||
)
|
||||
self.assertEqual(len(response.data.series), 2)
|
||||
self.assertEqual(len(response.data.series[0].points), count)
|
||||
self.assertLessEqual(count, response.meta.maximum_points)
|
||||
self.assertEqual(response.meta.bucket_seconds, seconds)
|
||||
self.assertEqual(response.meta.database_query_count, 1)
|
||||
|
||||
def test_zero_null_and_empty_bucket_semantics_remain_distinct(self):
|
||||
response = MonitoringService(FakeMonitoringRepository()).get_trends(
|
||||
MonitoringTrendQuery(scope="fleet", range="6h"),
|
||||
include_location_scopes=False,
|
||||
generated_at=NOW,
|
||||
)
|
||||
points = response.data.series[0].points
|
||||
self.assertEqual(points[0].value, 0)
|
||||
self.assertEqual(points[0].metric_sample_count, 1)
|
||||
self.assertIsNone(points[1].value)
|
||||
self.assertEqual(points[1].observation_count, 1)
|
||||
self.assertEqual(points[1].metric_sample_count, 0)
|
||||
self.assertIsNone(points[2].value)
|
||||
self.assertEqual(points[2].observation_count, 0)
|
||||
|
||||
def test_location_options_and_attribution_are_permission_aware(self):
|
||||
repository = FakeMonitoringRepository(trend_rows(label="North Site"))
|
||||
response = MonitoringService(repository).get_trends(
|
||||
MonitoringTrendQuery(scope="site", scope_id=uuid4(), range="6h"),
|
||||
include_location_scopes=True,
|
||||
generated_at=NOW,
|
||||
)
|
||||
self.assertEqual(response.meta.attribution, "current_placement_cohort")
|
||||
self.assertEqual(response.meta.database_query_count, 2)
|
||||
self.assertEqual(repository.option_calls, 1)
|
||||
self.assertEqual(len(response.data.available_scopes.sites), 1)
|
||||
repository = FakeMonitoringRepository()
|
||||
response = MonitoringService(repository).get_trends(
|
||||
MonitoringTrendQuery(scope="fleet", range="6h"),
|
||||
include_location_scopes=False,
|
||||
generated_at=NOW,
|
||||
)
|
||||
self.assertEqual(response.data.available_scopes.sites, [])
|
||||
self.assertEqual(repository.option_calls, 0)
|
||||
|
||||
|
||||
class MonitoringRouterTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.repository = FakeMonitoringRepository()
|
||||
self.service = MonitoringService(self.repository)
|
||||
self.app = FastAPI()
|
||||
self.app.include_router(monitoring_router)
|
||||
self.app.dependency_overrides[get_monitoring_service] = lambda: self.service
|
||||
self.client = TestClient(self.app)
|
||||
|
||||
def tearDown(self):
|
||||
self.client.close()
|
||||
|
||||
def authenticate(self, permissions):
|
||||
self.app.dependency_overrides[get_current_user] = lambda: {
|
||||
"id": str(uuid4()), "role": "Custom", "permissions": list(permissions)
|
||||
}
|
||||
|
||||
def test_authentication_permission_and_location_scope_boundaries(self):
|
||||
self.assertEqual(self.client.get("/api/monitoring/trends").status_code, 401)
|
||||
self.authenticate(())
|
||||
self.assertEqual(self.client.get("/api/monitoring/trends").status_code, 403)
|
||||
self.authenticate(("miners.view",))
|
||||
self.assertEqual(self.client.get("/api/monitoring/trends").status_code, 200)
|
||||
response = self.client.get(
|
||||
"/api/monitoring/trends",
|
||||
params={"scope": "site", "scope_id": uuid4(), "range": "6h"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
self.assertEqual(self.repository.trend_calls[-1]["scope"], "fleet")
|
||||
|
||||
def test_typed_validation_rejects_unbounded_or_ambiguous_queries(self):
|
||||
self.authenticate(("miners.view", "locations.view"))
|
||||
cases = (
|
||||
{"scope": "fleet", "scope_id": uuid4()},
|
||||
{"scope": "miner"},
|
||||
{"scope": "miner", "scope_id": "bad"},
|
||||
{"scope": "fleet", "range": "30d"},
|
||||
{"scope": "anything"},
|
||||
{"scope": "fleet", "resolution": "1 second"},
|
||||
)
|
||||
for query in cases[:-1]:
|
||||
with self.subTest(query=query):
|
||||
self.assertEqual(
|
||||
self.client.get("/api/monitoring/trends", params=query).status_code,
|
||||
422,
|
||||
)
|
||||
response = self.client.get("/api/monitoring/trends", params=cases[-1])
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertNotIn("resolution", self.repository.trend_calls[-1])
|
||||
|
||||
def test_typed_envelope_returns_both_series_and_no_store(self):
|
||||
self.authenticate(("miners.view",))
|
||||
response = self.client.get("/api/monitoring/trends?scope=fleet&range=6h")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(
|
||||
self.repository.trend_calls[-1]["window_end"]
|
||||
- self.repository.trend_calls[-1]["window_start"],
|
||||
timedelta(hours=6),
|
||||
)
|
||||
self.assertEqual(response.headers["cache-control"], "no-store")
|
||||
payload = response.json()
|
||||
self.assertEqual([item["metric"] for item in payload["data"]["series"]],
|
||||
["hashrate", "power"])
|
||||
self.assertEqual(payload["meta"]["source"], "raw_snapshots")
|
||||
self.assertEqual(payload["meta"]["completeness"], "not_guaranteed")
|
||||
self.assertIsNone(payload["error"])
|
||||
|
||||
|
||||
class MonitoringRepositoryIntegrationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.connection = psycopg.connect(
|
||||
host=settings.postgres_host,
|
||||
port=settings.postgres_port,
|
||||
dbname=settings.postgres_db,
|
||||
user=settings.postgres_user,
|
||||
password=settings.postgres_password,
|
||||
)
|
||||
self.repository = MonitoringRepository(connection=self.connection)
|
||||
|
||||
def tearDown(self):
|
||||
self.connection.rollback()
|
||||
self.connection.close()
|
||||
|
||||
def test_location_scope_options_execute_and_order_deterministically(self):
|
||||
site_alpha, site_beta = uuid4(), uuid4()
|
||||
room_alpha, room_beta = uuid4(), uuid4()
|
||||
suffix = uuid4().hex[:10]
|
||||
with self.connection.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO sites (id, name) VALUES (%s, %s), (%s, %s)",
|
||||
(site_alpha, "Scope Alpha " + suffix,
|
||||
site_beta, "scope Beta " + suffix),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO rooms (id, site_id, name) VALUES "
|
||||
"(%s, %s, %s), (%s, %s, %s)",
|
||||
(room_alpha, site_alpha, "Scope Alpha Room " + suffix,
|
||||
room_beta, site_beta, "scope Beta Room " + suffix),
|
||||
)
|
||||
|
||||
rows = self.repository.list_location_scope_options()
|
||||
site_rows = [row for row in rows if row["option_type"] == "site"]
|
||||
room_rows = [row for row in rows if row["option_type"] == "room"]
|
||||
|
||||
self.assertEqual(
|
||||
[row["option_type"] for row in rows],
|
||||
["site"] * len(site_rows) + ["room"] * len(room_rows),
|
||||
)
|
||||
ordering = lambda row: (row["name"].lower(), str(row["id"]))
|
||||
self.assertEqual(site_rows, sorted(site_rows, key=ordering))
|
||||
self.assertEqual(room_rows, sorted(room_rows, key=ordering))
|
||||
|
||||
by_id = {row["id"]: row for row in rows}
|
||||
self.assertIsNone(by_id[site_alpha]["site_id"])
|
||||
self.assertIsNone(by_id[site_beta]["site_id"])
|
||||
self.assertEqual(by_id[room_alpha]["site_id"], site_alpha)
|
||||
self.assertEqual(by_id[room_beta]["site_id"], site_beta)
|
||||
|
||||
def test_per_miner_first_math_counts_and_current_location_precedence(self):
|
||||
site_a, site_b, room_a = uuid4(), uuid4(), uuid4()
|
||||
miner_a, miner_b = uuid4(), uuid4()
|
||||
suffix = uuid4().hex[:10]
|
||||
with self.connection.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO sites (id, name) VALUES (%s, %s), (%s, %s)",
|
||||
(site_a, "Trend A " + suffix, site_b, "Trend B " + suffix),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO rooms (id, site_id, name) VALUES (%s, %s, %s)",
|
||||
(room_a, site_a, "Trend Room " + suffix),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO miners (id, site_id, room_id, asset_tag, lifecycle_status) "
|
||||
"VALUES (%s, %s, NULL, %s, 'active'), "
|
||||
"(%s, %s, %s, %s, 'active')",
|
||||
(miner_a, site_b, "TREND-A-" + suffix,
|
||||
miner_b, site_a, room_a, "TREND-B-" + suffix),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO location_positions "
|
||||
"(room_id, room_name, rack_name, shelf_name, position_index, miner_id) "
|
||||
"VALUES (%s, %s, %s, '1', 1, %s)",
|
||||
(room_a, "Trend Room " + suffix, "Rack " + suffix, miner_a),
|
||||
)
|
||||
observed = datetime(2026, 8, 17, 11, 56, tzinfo=timezone.utc)
|
||||
cur.execute(
|
||||
"INSERT INTO miner_status_snapshots "
|
||||
"(miner_id, observed_at, mhs_av, power_w) VALUES "
|
||||
"(%s, %s, 100000000, 1000), (%s, %s, 200000000, NULL), "
|
||||
"(%s, %s, 0, 0), (%s, %s, NULL, 3000)",
|
||||
(miner_a, observed, miner_a, observed + timedelta(minutes=1),
|
||||
miner_b, observed, miner_b, observed + timedelta(minutes=1)),
|
||||
)
|
||||
rows = self.repository.read_trends(
|
||||
scope="site", scope_id=site_a,
|
||||
window_start=datetime(2026, 8, 17, 6, 5, tzinfo=timezone.utc),
|
||||
window_end=datetime(2026, 8, 17, 12, 5, tzinfo=timezone.utc),
|
||||
bucket_width=timedelta(minutes=5),
|
||||
)
|
||||
point = next(row for row in rows if row["bucket_start"].hour == 11
|
||||
and row["bucket_start"].minute == 55)
|
||||
self.assertEqual(float(point["hashrate_ths"]), 150.0)
|
||||
self.assertEqual(float(point["power_w"]), 2500.0)
|
||||
self.assertEqual(point["observation_count"], 4)
|
||||
self.assertEqual(point["hashrate_sample_count"], 3)
|
||||
self.assertEqual(point["power_sample_count"], 3)
|
||||
self.assertEqual(point["observed_miner_count"], 2)
|
||||
self.assertEqual(point["hashrate_contributing_miner_count"], 2)
|
||||
self.assertEqual(point["scoped_miner_count"], 2)
|
||||
empty = next(row for row in rows if row["bucket_start"].hour == 11
|
||||
and row["bucket_start"].minute == 50)
|
||||
self.assertIsNone(empty["hashrate_ths"])
|
||||
self.assertEqual(empty["observation_count"], 0)
|
||||
self.assertEqual(len(rows), 72)
|
||||
|
||||
def test_utc_bucket_is_independent_of_session_timezone(self):
|
||||
miner_id = uuid4()
|
||||
with self.connection.cursor() as cur:
|
||||
cur.execute("SET LOCAL TIME ZONE 'Pacific/Honolulu'")
|
||||
cur.execute(
|
||||
"INSERT INTO miners (id, asset_tag, lifecycle_status) "
|
||||
"VALUES (%s, %s, 'active')",
|
||||
(miner_id, "TZ-" + uuid4().hex[:10]),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO miner_status_snapshots "
|
||||
"(miner_id, observed_at, mhs_av) VALUES (%s, %s, 1000000)",
|
||||
(miner_id, datetime.fromisoformat("2026-08-17T07:01:00-05:00")),
|
||||
)
|
||||
rows = self.repository.read_trends(
|
||||
scope="miner", scope_id=miner_id,
|
||||
window_start=datetime(2026, 8, 17, 6, 5, tzinfo=timezone.utc),
|
||||
window_end=datetime(2026, 8, 17, 12, 5, tzinfo=timezone.utc),
|
||||
bucket_width=timedelta(minutes=5),
|
||||
)
|
||||
value = next(row for row in rows
|
||||
if row["bucket_start"].astimezone(timezone.utc).hour == 12)
|
||||
self.assertEqual(float(value["hashrate_ths"]), 1.0)
|
||||
|
||||
def test_representative_raw_query_is_bounded(self):
|
||||
prefix = "BENCH-" + uuid4().hex[:8]
|
||||
with self.connection.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO miners (asset_tag, lifecycle_status) "
|
||||
"SELECT %s || '-' || value, 'active' "
|
||||
"FROM generate_series(1, 500) AS value",
|
||||
(prefix,),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO miner_status_snapshots "
|
||||
"(miner_id, observed_at, mhs_av, power_w) "
|
||||
"SELECT miners.id, %s::timestamptz - value * interval '1 hour', "
|
||||
"100000000, 3200 FROM miners "
|
||||
"CROSS JOIN generate_series(0, 167) AS value "
|
||||
"WHERE miners.asset_tag LIKE %s",
|
||||
(NOW, prefix + "%"),
|
||||
)
|
||||
started = time.monotonic()
|
||||
rows = self.repository.read_trends(
|
||||
scope="fleet", scope_id=None,
|
||||
window_start=datetime(2026, 8, 10, 13, 0, tzinfo=timezone.utc),
|
||||
window_end=datetime(2026, 8, 17, 13, 0, tzinfo=timezone.utc),
|
||||
bucket_width=timedelta(hours=1),
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
self.assertEqual(len(rows), 168)
|
||||
self.assertLess(elapsed, 15.0)
|
||||
print(f"Monitoring synthetic benchmark: 84,000 rows, 168 points, {elapsed:.3f}s")
|
||||
|
||||
|
||||
class MonitoringFrontendTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = FastAPI()
|
||||
self.app.include_router(monitoring_frontend_router)
|
||||
self.client = TestClient(self.app)
|
||||
|
||||
def tearDown(self):
|
||||
self.client.close()
|
||||
|
||||
def authenticate(self, permissions):
|
||||
self.app.dependency_overrides[get_current_user] = lambda: {
|
||||
"id": str(uuid4()), "username": "reader", "display_name": "Reader",
|
||||
"role": "Custom", "permissions": list(permissions),
|
||||
}
|
||||
|
||||
def test_page_requires_scope_aligned_permissions_and_renders_shell(self):
|
||||
self.assertEqual(self.client.get("/monitoring/analytics").status_code, 401)
|
||||
self.authenticate(("miners.view",))
|
||||
response = self.client.get("/monitoring/analytics")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertIn('data-view="analytics"', response.text)
|
||||
self.assertIn("/assets/minerops-monitoring-trends.js", response.text)
|
||||
self.assertNotIn('<option value="site">', response.text)
|
||||
response = self.client.get(
|
||||
"/monitoring/analytics",
|
||||
params={"scope": "site", "scope_id": uuid4(), "range": "24h"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
self.authenticate(("miners.view", "locations.view"))
|
||||
self.assertEqual(self.client.get(
|
||||
"/monitoring/analytics",
|
||||
params={"scope": "site", "scope_id": uuid4(), "range": "24h"},
|
||||
).status_code, 200)
|
||||
|
||||
def test_script_is_one_request_read_only_safe_dom_and_accessible(self):
|
||||
source = (STATIC_ROOT / "monitoring-trends.js").read_text(encoding="utf-8")
|
||||
self.assertEqual(source.count('fetch(url,'), 1)
|
||||
self.assertIn("return el.site.value || state.scopeId", source)
|
||||
self.assertIn("return el.room.value || state.scopeId", source)
|
||||
self.assertNotIn("setInterval", source)
|
||||
for method in ('method: "POST"', 'method: "PATCH"', 'method: "PUT"',
|
||||
'method: "DELETE"'):
|
||||
self.assertNotIn(method, source)
|
||||
for unsafe in ("innerHTML", "outerHTML", "insertAdjacentHTML",
|
||||
"document.write", "eval(", "Function("):
|
||||
self.assertNotIn(unsafe, source)
|
||||
for expected in ("textContent", "tabindex", "aria-label", "View ",
|
||||
"Gaps indicate", "prefers-reduced-motion"):
|
||||
combined = source + (STATIC_ROOT / "monitoring-trends.css").read_text()
|
||||
self.assertIn(expected, combined)
|
||||
for forbidden in ("threshold", "healthy range", "kWh", "30d"):
|
||||
self.assertNotIn(forbidden, source)
|
||||
|
||||
def test_navigation_identity_and_miner_detail_use_canonical_durable_link(self):
|
||||
self.assertTrue(protected_page("/monitoring/analytics"))
|
||||
self.assertEqual(required_page_permission("/monitoring/analytics"), "miners.view")
|
||||
detail_html = (APP_ROOT / "frontend" / "miner_detail_view.py").read_text()
|
||||
detail_js = (STATIC_ROOT / "miner-detail.js").read_text()
|
||||
self.assertIn("View trends", detail_html)
|
||||
self.assertIn('scope: "miner", scope_id: data.miner.id, range: "24h"', detail_js)
|
||||
self.assertIn('"/monitoring/analytics?"', detail_js)
|
||||
|
||||
|
||||
class MonitoringArchitectureTests(unittest.TestCase):
|
||||
def test_layering_routes_and_sql_are_bounded(self):
|
||||
for relative in ("monitoring/router.py", "monitoring/service.py",
|
||||
"monitoring/frontend.py"):
|
||||
source = (APP_ROOT / relative).read_text(encoding="utf-8")
|
||||
modules = []
|
||||
for node in ast.walk(ast.parse(source)):
|
||||
if isinstance(node, ast.Import):
|
||||
modules.extend(alias.name for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
modules.append(node.module)
|
||||
self.assertFalse(any(name.startswith(("psycopg", "asyncpg", "sqlalchemy"))
|
||||
for name in modules))
|
||||
for statement in ("SELECT ", "INSERT ", "UPDATE ", "DELETE "):
|
||||
self.assertNotIn(statement, source.upper())
|
||||
service = (APP_ROOT / "monitoring" / "service.py").read_text()
|
||||
repository = (APP_ROOT / "monitoring" / "repository.py").read_text()
|
||||
self.assertNotIn("fastapi", service.lower())
|
||||
self.assertEqual(repository.count("cur.execute("), 2)
|
||||
self.assertIn("miner_status_snapshots", TREND_SQL)
|
||||
self.assertNotIn("miner_status_hourly_rollups", TREND_SQL)
|
||||
self.assertNotIn("miner_status_daily_rollups", TREND_SQL)
|
||||
self.assertIn("date_bin", TREND_SQL)
|
||||
self.assertIn("%s", TREND_SQL)
|
||||
self.assertEqual([(route.path, route.methods) for route in monitoring_router.routes],
|
||||
[("/api/monitoring/trends", {"GET"})])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user