security: contain physical miner commands in Stage A
Disable physical control through legacy HTTP/M30 producers, typed admission, queue dispatch and upgraded Windows executors. Restrict diagnostic transports to exact read-only operations, with ordinal PowerShell validation before sockets. Retire unsupported pending/claimed work with atomic, idempotent security audits. Preserve prior delivery evidence and report unknown outcomes after a prior claim, including explicitly repaired pending rows. Include containment, cancellation, concurrency and malformed-payload regressions and the old-Agent cutover runbook. Validation: independent final application-security revalidation PASS. Two unchanged baseline source-assertion failures are documented as exceptions. Production release still requires native Windows validation and verified fencing/stop/drain/upgrade; server cancellation alone cannot revoke a cached command held by an old Agent.
This commit is contained in:
lussierj committed
1 parent
2d89e80410
commit
e898f283ab
28 files changed
+2377
-2379
No files matched your search
@@ -0,0 +1,35 @@
|
||||
"""Stage A containment: no current payload authorizes a physical write.
|
||||
|
||||
There is deliberately no environment-variable override. Re-enabling physical
|
||||
control requires a separately reviewed identity-binding protocol.
|
||||
"""
|
||||
|
||||
PHYSICAL_CONTROL_DISABLED = "physical_control_disabled_stage_a"
|
||||
UNSUPPORTED_COMMAND = "unsupported_command_schema_stage_a"
|
||||
READ_ONLY_COMMAND_TYPES = (
|
||||
"poll_known_miners_now",
|
||||
"discover_new_miners",
|
||||
"enrich_match_discovery",
|
||||
)
|
||||
|
||||
|
||||
def rejection_reason(command_type: str) -> str | None:
|
||||
if command_type == "miner_control":
|
||||
return PHYSICAL_CONTROL_DISABLED
|
||||
if command_type not in READ_ONLY_COMMAND_TYPES:
|
||||
return UNSUPPORTED_COMMAND
|
||||
return None
|
||||
|
||||
|
||||
def rejection_execution_outcome(row: dict) -> str:
|
||||
"""A current pending state alone says nothing about earlier delivery.
|
||||
|
||||
The repository supplies an explicit pre-transition evidence check while
|
||||
holding the queue row lock. Missing/ambiguous checks and result submissions
|
||||
are unknown. This describes queue delivery, never proof of physical outcome.
|
||||
"""
|
||||
if (row.get("previous_status") == "pending"
|
||||
and row.get("no_delivery_evidence") is True
|
||||
and row.get("delivery_reported", False) is False):
|
||||
return "not_dispatched"
|
||||
return "unknown"
|
||||
@@ -8,6 +8,7 @@ from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from app.agent_commands.exceptions import AgentCommandConflict, AgentCommandNotFound
|
||||
from app.agent_commands.policy import READ_ONLY_COMMAND_TYPES, rejection_execution_outcome, rejection_reason
|
||||
from app.config import settings
|
||||
|
||||
|
||||
@@ -19,6 +20,7 @@ WITH candidates AS (
|
||||
FROM minerops_agent_command_queue
|
||||
WHERE agent_id = %s
|
||||
AND command_status = 'pending'
|
||||
AND command_type = ANY(%s)
|
||||
ORDER BY created_at, id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT %s
|
||||
@@ -32,6 +34,70 @@ RETURNING queue.id AS command_id, queue.command_type,
|
||||
queue.request_payload, queue.created_at
|
||||
"""
|
||||
|
||||
# Inspect existing durable history before changing status/completed_at/error.
|
||||
# A linked non-creation audit is conservatively ambiguous, even if timestamps
|
||||
# were lost. Legacy results need not agree on Agent ID to prevent a false claim
|
||||
# of non-delivery. Missing tables/permissions fail the transaction closed.
|
||||
NO_DELIVERY_EVIDENCE_SQL = """
|
||||
q.created_at IS NOT NULL
|
||||
AND q.claimed_at IS NULL AND q.completed_at IS NULL
|
||||
AND q.result_payload IS NULL AND COALESCE(q.error_message, '') = ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM minerops_agent_command_results r
|
||||
WHERE r.command_id = q.id
|
||||
OR (r.agent_id = q.agent_id AND NULLIF(btrim(r.command_id), '') IS NULL)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM audit_logs a
|
||||
WHERE a.action <> 'miner.control.queued'
|
||||
AND (a.old_value_json ->> 'command_id' = q.id
|
||||
OR a.new_value_json ->> 'command_id' = q.id
|
||||
OR a.old_value_json ->> 'operation_id' = q.id
|
||||
OR a.new_value_json ->> 'operation_id' = q.id
|
||||
OR (a.entity_type = 'agent_command' AND a.entity_id::text = q.id))
|
||||
)
|
||||
"""
|
||||
|
||||
RETIRE_COMMANDS_SQL = f"""
|
||||
WITH candidates AS (
|
||||
SELECT q.id, q.command_status, ({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
|
||||
FROM minerops_agent_command_queue q
|
||||
WHERE agent_id = %s AND command_status IN ('pending', 'claimed')
|
||||
AND NOT (command_type = ANY(%s))
|
||||
ORDER BY created_at, id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1000
|
||||
)
|
||||
UPDATE minerops_agent_command_queue AS queue
|
||||
SET command_status = 'cancelled', completed_at = COALESCE(queue.completed_at, now()),
|
||||
error_message = concat_ws(chr(10), NULLIF(queue.error_message, ''), CASE WHEN queue.command_type = 'miner_control'
|
||||
THEN 'physical_control_disabled_stage_a'
|
||||
ELSE 'unsupported_command_schema_stage_a' END)
|
||||
FROM candidates WHERE queue.id = candidates.id
|
||||
RETURNING queue.id, queue.agent_id, queue.command_type,
|
||||
candidates.command_status AS previous_status, candidates.no_delivery_evidence, queue.error_message
|
||||
"""
|
||||
|
||||
RETIRE_SCHEDULE_COMMANDS_SQL = f"""
|
||||
WITH candidates AS (
|
||||
SELECT q.id, q.command_status, ({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
|
||||
FROM minerops_agent_command_queue q
|
||||
WHERE agent_id = %s AND request_payload ->> 'scheduled_action_id' = %s
|
||||
AND command_status IN ('pending', 'claimed')
|
||||
AND NOT (command_type = ANY(%s))
|
||||
ORDER BY id
|
||||
FOR UPDATE
|
||||
)
|
||||
UPDATE minerops_agent_command_queue AS queue
|
||||
SET command_status = 'cancelled', completed_at = COALESCE(queue.completed_at, now()),
|
||||
error_message = concat_ws(chr(10), NULLIF(queue.error_message, ''), CASE WHEN queue.command_type = 'miner_control'
|
||||
THEN 'physical_control_disabled_stage_a'
|
||||
ELSE 'unsupported_command_schema_stage_a' END)
|
||||
FROM candidates WHERE queue.id = candidates.id
|
||||
RETURNING queue.id, queue.agent_id, queue.command_type,
|
||||
candidates.command_status AS previous_status, candidates.no_delivery_evidence
|
||||
"""
|
||||
|
||||
|
||||
def default_connection_factory() -> psycopg.Connection:
|
||||
return psycopg.connect(
|
||||
@@ -54,7 +120,8 @@ class AgentCommandRepository:
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
with connection.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(CLAIM_COMMANDS_SQL, (agent_id, limit))
|
||||
self._retire_unsupported(cur, agent_id)
|
||||
cur.execute(CLAIM_COMMANDS_SQL, (agent_id, list(READ_ONLY_COMMAND_TYPES), limit))
|
||||
rows = [dict(row) for row in cur.fetchall()]
|
||||
connection.commit()
|
||||
return rows
|
||||
@@ -64,6 +131,105 @@ class AgentCommandRepository:
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
@staticmethod
|
||||
def _audit_rejection(
|
||||
cur, row: dict, *, actor_user_id: str | None = None,
|
||||
source: str = "agent_poll",
|
||||
) -> None:
|
||||
# Queue IDs can be legacy text, not UUIDs. Preserve them in JSON rather
|
||||
# than coercing them into audit_logs.entity_id. Never log raw payloads.
|
||||
cur.execute(
|
||||
"""INSERT INTO audit_logs
|
||||
(actor_user_id, action, entity_type, old_value_json, new_value_json)
|
||||
VALUES (%s, 'agent.command.rejected', 'agent_command', %s, %s)""",
|
||||
(actor_user_id, Jsonb({"command_id": row["id"], "agent_id": row["agent_id"],
|
||||
"status": row["previous_status"]}),
|
||||
Jsonb({"status": "cancelled", "reason": rejection_reason(row["command_type"]),
|
||||
"source": source,
|
||||
"execution_outcome": rejection_execution_outcome(row)})),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _retire_unsupported(
|
||||
cls, cur, agent_id: str, *, source: str = "agent_poll",
|
||||
actor_user_id: str | None = None,
|
||||
) -> int:
|
||||
cur.execute(RETIRE_COMMANDS_SQL, (agent_id, list(READ_ONLY_COMMAND_TYPES)))
|
||||
rows = list(cur.fetchall())
|
||||
for row in rows:
|
||||
cls._audit_rejection(cur, row, source=source, actor_user_id=actor_user_id)
|
||||
return len(rows)
|
||||
|
||||
def retire_unsupported(self, agent_id: str, *, actor_user_id: str) -> int:
|
||||
"""Audited cutover batch, including offline Agents; never claim any work.
|
||||
|
||||
A zero batch can mean locked rows were skipped. Operators must separately
|
||||
verify no unsupported live rows remain before declaring a drain complete.
|
||||
"""
|
||||
if not actor_user_id:
|
||||
raise ValueError("Cutover retirement requires an accountable operator actor")
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
with connection.cursor(row_factory=dict_row) as cur:
|
||||
count = self._retire_unsupported(cur, agent_id, source="stage_a_cutover",
|
||||
actor_user_id=actor_user_id)
|
||||
connection.commit()
|
||||
return count
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def retire_schedule(self, schedule_id: str, *, actor_user_id: str, operation: str) -> int:
|
||||
"""Cancel live unsupported work and retain schedule history atomically.
|
||||
|
||||
The stored schedule determines Agent scope. Delete retires the schedule;
|
||||
neither operation erases evidence or changes terminal command history.
|
||||
"""
|
||||
if not actor_user_id or operation not in {"cancel", "delete"}:
|
||||
raise ValueError("An authenticated actor and cancel/delete operation are required")
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
with connection.cursor(row_factory=dict_row) as cur:
|
||||
# Serialize retries and cancel/delete workers for this schedule.
|
||||
cur.execute(
|
||||
"""SELECT id, agent_id, status FROM minerops_scheduled_bulk_actions
|
||||
WHERE id = %s FOR UPDATE""", (schedule_id,),
|
||||
)
|
||||
schedule = cur.fetchone()
|
||||
if not schedule:
|
||||
raise AgentCommandNotFound("Scheduled action not found")
|
||||
cur.execute(RETIRE_SCHEDULE_COMMANDS_SQL,
|
||||
(schedule["agent_id"], schedule_id, list(READ_ONLY_COMMAND_TYPES)))
|
||||
rows = list(cur.fetchall())
|
||||
for row in rows:
|
||||
self._audit_rejection(cur, row, actor_user_id=actor_user_id,
|
||||
source=f"schedule_{operation}")
|
||||
if schedule["status"] in {"pending", "queued", "running"}:
|
||||
cur.execute(
|
||||
"""UPDATE minerops_scheduled_bulk_actions
|
||||
SET status = 'cancelled', updated_at = now(),
|
||||
last_error = concat_ws(chr(10), NULLIF(last_error, ''),
|
||||
'physical_control_disabled_stage_a')
|
||||
WHERE id = %s""", (schedule_id,),
|
||||
)
|
||||
cur.execute(
|
||||
"""INSERT INTO audit_logs
|
||||
(actor_user_id, action, entity_type, old_value_json, new_value_json)
|
||||
VALUES (%s, 'scheduled_action.retired', 'scheduled_action', %s, %s)""",
|
||||
(actor_user_id, Jsonb(dict(schedule)),
|
||||
Jsonb({"status": "cancelled", "operation": operation,
|
||||
"reason": "physical_control_disabled_stage_a"})),
|
||||
)
|
||||
connection.commit()
|
||||
return len(rows)
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def get_command(self, command_id: str) -> dict | None:
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
@@ -91,6 +257,32 @@ class AgentCommandRepository:
|
||||
connection = self._connection_factory()
|
||||
try:
|
||||
with connection.cursor(row_factory=dict_row) as cur:
|
||||
# Enforce the same boundary on results: an old claimed entry
|
||||
# cannot be promoted by an old Agent, retry, or direct caller.
|
||||
cur.execute(
|
||||
f"""SELECT q.id, q.agent_id, q.command_type, q.command_status AS previous_status,
|
||||
({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
|
||||
FROM minerops_agent_command_queue q
|
||||
WHERE q.id = %s AND q.agent_id = %s FOR UPDATE""",
|
||||
(command_id, agent_id),
|
||||
)
|
||||
existing = cur.fetchone()
|
||||
if existing and rejection_reason(existing["command_type"]):
|
||||
reason = rejection_reason(existing["command_type"])
|
||||
if existing["previous_status"] in {"pending", "claimed"}:
|
||||
cur.execute(
|
||||
"""UPDATE minerops_agent_command_queue
|
||||
SET command_status = 'cancelled',
|
||||
error_message = concat_ws(chr(10), NULLIF(error_message, ''), %s::text),
|
||||
completed_at = COALESCE(completed_at, now()) WHERE id = %s""",
|
||||
(reason, command_id),
|
||||
)
|
||||
# A result submission itself reports possible delivery, even
|
||||
# when earlier durable claim metadata is absent.
|
||||
existing["delivery_reported"] = True
|
||||
self._audit_rejection(cur, existing, source="agent_result")
|
||||
connection.commit()
|
||||
raise AgentCommandConflict(reason)
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE minerops_agent_command_queue
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
"""HTTP boundary for authenticated Agent command transport."""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
@@ -18,8 +16,6 @@ from app.agent_commands.models import (
|
||||
)
|
||||
from app.agent_commands.service import AgentCommandService
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/agent/commands", tags=["agent-commands"])
|
||||
AgentIdentity = Annotated[str, Depends(require_agent_identity)]
|
||||
AgentService = Annotated[AgentCommandService, Depends(get_agent_command_service)]
|
||||
@@ -29,14 +25,7 @@ AgentService = Annotated[AgentCommandService, Depends(get_agent_command_service)
|
||||
def claim_pending_commands(
|
||||
request: Request, agent_id: AgentIdentity, service: AgentService
|
||||
) -> AgentCommandListResponse:
|
||||
before_claim = getattr(
|
||||
request.app.state, "agent_command_before_claim", None
|
||||
)
|
||||
if callable(before_claim):
|
||||
try:
|
||||
before_claim()
|
||||
except Exception:
|
||||
logger.exception("Scheduled command materialization failed before Agent claim")
|
||||
del request
|
||||
return AgentCommandListResponse(commands=service.claim(agent_id))
|
||||
|
||||
|
||||
|
||||
@@ -2,19 +2,16 @@
|
||||
|
||||
import re
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.agent_commands.exceptions import (
|
||||
AgentCommandConflict,
|
||||
AgentCommandNotFound,
|
||||
AgentResultValidationError,
|
||||
)
|
||||
from app.agent_commands.models import (
|
||||
AgentCommand,
|
||||
AgentCommandResultRequest,
|
||||
MinerControlResultPayload,
|
||||
)
|
||||
from app.agent_commands.repository import AgentCommandRepository
|
||||
from app.agent_commands.policy import rejection_reason
|
||||
|
||||
|
||||
class AgentCommandService:
|
||||
@@ -25,6 +22,7 @@ class AgentCommandService:
|
||||
return [
|
||||
AgentCommand.model_validate(row)
|
||||
for row in self.repository.claim_pending(agent_id)
|
||||
if rejection_reason(row["command_type"]) is None
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@@ -42,6 +40,14 @@ class AgentCommandService:
|
||||
raise AgentCommandNotFound("Agent command not found")
|
||||
if command["agent_id"] != agent_id:
|
||||
raise AgentCommandConflict("Agent command belongs to another Agent")
|
||||
reason = rejection_reason(command["command_type"])
|
||||
if reason:
|
||||
# Repository repeats this check under a row lock and records the
|
||||
# cancellation atomically; never accept an old physical result.
|
||||
self.repository.finish_command(
|
||||
request.command_id, agent_id, "cancelled", {}, reason
|
||||
)
|
||||
raise AgentCommandConflict(reason)
|
||||
if command["command_status"] != "claimed":
|
||||
raise AgentCommandConflict(
|
||||
f"Agent command is {command['command_status']}, not claimed"
|
||||
@@ -49,34 +55,6 @@ class AgentCommandService:
|
||||
|
||||
result_payload = request.result_payload
|
||||
error_message = self._legacy_error(request.error_message)
|
||||
if command["command_type"] == "miner_control":
|
||||
try:
|
||||
result = MinerControlResultPayload.model_validate(request.result_payload)
|
||||
except ValidationError as exc:
|
||||
raise AgentResultValidationError(
|
||||
"Invalid typed miner-control result"
|
||||
) from exc
|
||||
if request.status == "completed" and not result.command_accepted:
|
||||
raise AgentResultValidationError(
|
||||
"Completed miner-control results must be accepted"
|
||||
)
|
||||
if request.status == "failed" and result.command_accepted:
|
||||
raise AgentResultValidationError(
|
||||
"Failed miner-control results cannot be accepted"
|
||||
)
|
||||
action = (command.get("request_payload") or {}).get("action")
|
||||
if (
|
||||
action in {"suspend", "resume"}
|
||||
and result.command_accepted
|
||||
and result.verification_status != "unconfirmed"
|
||||
):
|
||||
raise AgentResultValidationError(
|
||||
"Accepted suspend and resume results must remain unconfirmed"
|
||||
)
|
||||
result_payload = result.model_dump(mode="json")
|
||||
error_message = (
|
||||
result.failure_code.replace("_", " ") if result.failure_code else None
|
||||
)
|
||||
|
||||
self.repository.finish_command(
|
||||
request.command_id,
|
||||
|
||||
@@ -708,6 +708,8 @@ def machine_agent_request(path: str) -> bool:
|
||||
"/api/agent/pool-snapshots",
|
||||
"/api/agent/commands/pending",
|
||||
"/api/agent/commands/result",
|
||||
"/api/agent/commands/pending-disabled-old",
|
||||
"/api/agent/commands/result-disabled-old",
|
||||
"/api/agent/discoveries",
|
||||
"/api/agent/miner-enrichment",
|
||||
"/api/agent/miner-enrichment-v2",
|
||||
@@ -760,6 +762,19 @@ def component_request(path: str) -> bool:
|
||||
|
||||
def required_api_permission(method: str, path: str) -> str | None:
|
||||
parts = path.split("/")
|
||||
if method == "POST" and path == "/api/dashboard/m30-command-test-lab/run":
|
||||
return "miners.control"
|
||||
if method == "POST" and path == "/api/tools/whatsminer-read-test":
|
||||
return "miners.control"
|
||||
if method == "POST" and path in {
|
||||
"/api/dashboard/scheduled-bulk-actions",
|
||||
"/api/dashboard/scheduled-bulk-actions/process-due",
|
||||
}:
|
||||
return "miners.control"
|
||||
if (method == "POST" and len(parts) == 6
|
||||
and parts[:4] == ["", "api", "dashboard", "scheduled-bulk-actions"]
|
||||
and parts[5] in {"cancel", "delete"}):
|
||||
return "miners.control"
|
||||
if (
|
||||
len(parts) >= 5
|
||||
and parts[:3] == ["", "api", "miners"]
|
||||
|
||||
+147
-1589
File diff suppressed because it is too large.
Load diff
@@ -1,7 +1,6 @@
|
||||
"""Business rules for typed, Agent-routed miner controls."""
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
from uuid import UUID
|
||||
|
||||
from app.miner_control.exceptions import (
|
||||
ControlledMinerNotFound,
|
||||
@@ -16,6 +15,7 @@ from app.miner_control.models import (
|
||||
MinerControlProjection,
|
||||
)
|
||||
from app.miner_control.repository import MinerControlRepository
|
||||
from app.agent_commands.policy import PHYSICAL_CONTROL_DISABLED
|
||||
|
||||
|
||||
KNOWN_MODES = {"low", "normal", "high"}
|
||||
@@ -29,33 +29,8 @@ class MinerControlService:
|
||||
|
||||
@staticmethod
|
||||
def _capability(context: dict) -> ControlCapability:
|
||||
if not context.get("agent_zone_id"):
|
||||
return ControlCapability(state="unavailable", reason="agent_zone_unassigned")
|
||||
if context.get("agent_zone_exists") is not True:
|
||||
return ControlCapability(state="unavailable", reason="agent_zone_missing")
|
||||
if context.get("agent_zone_enabled") is not True:
|
||||
return ControlCapability(state="unavailable", reason="agent_zone_disabled")
|
||||
if not context.get("agent_id"):
|
||||
return ControlCapability(state="unavailable", reason="agent_unavailable")
|
||||
if not context.get("current_ip"):
|
||||
return ControlCapability(state="unavailable", reason="current_ip_missing")
|
||||
if context.get("api3_supported") is False:
|
||||
return ControlCapability(state="unavailable", reason="api3_unsupported")
|
||||
if context.get("api3_supported") is not True:
|
||||
return ControlCapability(state="unknown", reason="api3_support_unknown")
|
||||
|
||||
payload = context.get("last_payload") or {}
|
||||
if payload.get("control_available") is False:
|
||||
reason = payload.get("control_unavailable_reason")
|
||||
return ControlCapability(
|
||||
state="unavailable",
|
||||
reason=reason if reason == "credentials_missing" else "agent_unavailable",
|
||||
)
|
||||
if payload.get("control_available") is not True:
|
||||
return ControlCapability(
|
||||
state="unknown", reason="agent_capability_unknown"
|
||||
)
|
||||
return ControlCapability(state="supported")
|
||||
# Credentials and Zone routing are not physical identity authorization.
|
||||
return ControlCapability(state="unavailable", reason=PHYSICAL_CONTROL_DISABLED)
|
||||
|
||||
@staticmethod
|
||||
def _operation(miner_id: UUID, row: dict) -> MinerControlOperation:
|
||||
@@ -120,50 +95,4 @@ class MinerControlService:
|
||||
) -> MinerControlAccepted:
|
||||
if not actor_user_id:
|
||||
raise MinerControlActorRequired("Authenticated actor UUID is required")
|
||||
context = self.repository.get_control_context(miner_id)
|
||||
if not context:
|
||||
raise ControlledMinerNotFound("Miner not found")
|
||||
capability = self._capability(context)
|
||||
if capability.state != "supported":
|
||||
raise MinerControlUnavailable(
|
||||
capability.reason or "control_capability_not_supported"
|
||||
)
|
||||
|
||||
runtime = str(context.get("runtime_state") or "").strip().lower()
|
||||
if action == "suspend" and runtime not in RUNNING_STATES:
|
||||
raise MinerControlUnavailable("miner_not_factually_running")
|
||||
if action == "resume" and runtime not in STOPPED_STATES:
|
||||
raise MinerControlUnavailable("miner_not_factually_stopped")
|
||||
if action == "performance_mode" and requested_value not in KNOWN_MODES:
|
||||
raise MinerControlUnavailable("invalid_performance_mode")
|
||||
|
||||
operation_id = str(uuid4())
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"miner_id": str(miner_id),
|
||||
"action": action,
|
||||
"requested_value": requested_value,
|
||||
"ip_address": context["current_ip"],
|
||||
}
|
||||
operation = self.repository.enqueue_with_audit(
|
||||
operation_id=operation_id,
|
||||
miner_id=miner_id,
|
||||
agent_id=context["agent_id"],
|
||||
actor_user_id=actor_user_id,
|
||||
actor_username=actor_username,
|
||||
request_payload=payload,
|
||||
audit_payload={
|
||||
"operation_id": operation_id,
|
||||
"miner_id": str(miner_id),
|
||||
"action": action,
|
||||
"requested_value": requested_value,
|
||||
},
|
||||
)
|
||||
return MinerControlAccepted(
|
||||
operation_id=operation["operation_id"],
|
||||
miner_id=miner_id,
|
||||
action=action,
|
||||
requested_value=requested_value,
|
||||
status=operation["status"],
|
||||
created_at=operation["created_at"],
|
||||
)
|
||||
raise MinerControlUnavailable(PHYSICAL_CONTROL_DISABLED)
|
||||
@@ -16,7 +16,7 @@ from app.agent_commands.exceptions import (
|
||||
AgentCommandConflict,
|
||||
AgentResultValidationError,
|
||||
)
|
||||
from app.agent_commands.models import AgentCommandResultRequest
|
||||
from app.agent_commands.models import AgentCommandResultRequest, MinerControlResultPayload
|
||||
from app.agent_commands.repository import CLAIM_COMMANDS_SQL
|
||||
from app.agent_commands.router import router
|
||||
from app.agent_commands.service import AgentCommandService
|
||||
@@ -38,14 +38,14 @@ class FakeRepository:
|
||||
def __init__(self, row=None):
|
||||
self.row = row or {
|
||||
"id": "command-1", "agent_id": "test-agent",
|
||||
"command_type": "miner_control", "command_status": "claimed",
|
||||
"command_type": "poll_known_miners_now", "command_status": "claimed",
|
||||
"request_payload": {"action": "suspend"},
|
||||
}
|
||||
self.finished = []
|
||||
|
||||
def claim_pending(self, agent_id, limit=5):
|
||||
return [{
|
||||
"command_id": "command-1", "command_type": "miner_control",
|
||||
"command_id": "command-1", "command_type": "poll_known_miners_now",
|
||||
"request_payload": {"action": "suspend"}, "created_at": NOW,
|
||||
}]
|
||||
|
||||
@@ -54,6 +54,8 @@ class FakeRepository:
|
||||
|
||||
def finish_command(self, *args):
|
||||
self.finished.append(args)
|
||||
if args[2] == "cancelled":
|
||||
raise AgentCommandConflict(args[4])
|
||||
|
||||
|
||||
def typed_result(**overrides):
|
||||
@@ -84,7 +86,7 @@ class AgentCommandServiceTests(unittest.TestCase):
|
||||
)
|
||||
AgentCommandService(repository).submit_result("test-agent", request)
|
||||
finished = repository.finished[0]
|
||||
self.assertIsNone(finished[4])
|
||||
self.assertIn("[address]", finished[4])
|
||||
self.assertNotIn("192.0.2.10", str(finished))
|
||||
|
||||
def test_wrong_agent_pending_and_terminal_results_are_rejected(self):
|
||||
@@ -111,8 +113,8 @@ class AgentCommandServiceTests(unittest.TestCase):
|
||||
command_id="command-1", status="completed",
|
||||
result_payload={**typed_result(), forbidden: "sensitive"},
|
||||
)
|
||||
with self.assertRaises(AgentResultValidationError):
|
||||
AgentCommandService(repository).submit_result("test-agent", request)
|
||||
with self.assertRaises(ValueError):
|
||||
MinerControlResultPayload.model_validate(request.result_payload)
|
||||
|
||||
def test_suspend_resume_cannot_claim_confirmation(self):
|
||||
for status in ("confirmed", "verification_failed"):
|
||||
@@ -123,8 +125,10 @@ class AgentCommandServiceTests(unittest.TestCase):
|
||||
verification_attempted=True, verification_status=status
|
||||
),
|
||||
)
|
||||
with self.assertRaises(AgentResultValidationError):
|
||||
AgentCommandService(FakeRepository()).submit_result("test-agent", request)
|
||||
repository = FakeRepository()
|
||||
repository.row["command_type"] = "miner_control"
|
||||
with self.assertRaisesRegex(AgentCommandConflict, "physical_control_disabled_stage_a"):
|
||||
AgentCommandService(repository).submit_result("test-agent", request)
|
||||
|
||||
|
||||
class FakeAgentAuthRepository:
|
||||
@@ -160,17 +164,26 @@ class AgentCommandRouterTests(unittest.TestCase):
|
||||
"X-MinerOps-Agent-Secret": "test-secret",
|
||||
}
|
||||
|
||||
def test_poll_requires_header_identity_and_ignores_query_spoofing(self):
|
||||
def test_poll_accepts_secret_only_legacy_agent_and_ignores_query_spoofing(self):
|
||||
before_claim_calls = []
|
||||
self.app.state.agent_command_before_claim = lambda: before_claim_calls.append(True)
|
||||
self.assertEqual(self.client.get("/api/agent/commands/pending").status_code, 401)
|
||||
self.assertEqual(
|
||||
self.client.get(
|
||||
"/api/agent/commands/pending",
|
||||
headers={"X-MinerOps-Agent-Secret": "test-secret"},
|
||||
).status_code,
|
||||
200,
|
||||
)
|
||||
self.assertEqual(
|
||||
self.client.get(
|
||||
"/api/agent/commands/pending?agent_id=other", headers=self.headers()
|
||||
).status_code,
|
||||
200,
|
||||
)
|
||||
self.assertEqual(before_claim_calls, [True])
|
||||
# Agent polling may claim already queued commands, but must never invoke
|
||||
# a legacy scheduler/materializer as a side effect.
|
||||
self.assertEqual(before_claim_calls, [])
|
||||
self.assertEqual(
|
||||
self.client.get(
|
||||
"/api/agent/commands/pending", headers=self.headers("other")
|
||||
@@ -214,9 +227,9 @@ class AgentSourceContractTests(unittest.TestCase):
|
||||
self.assertIn('.agent_zone_id', service)
|
||||
self.assertIn('"agent_zone_id"', (root / "config.example.json").read_text())
|
||||
self.assertNotIn("?agent_id=", service)
|
||||
self.assertIn('"get.miner.setting"', control)
|
||||
self.assertIn("$attempt -le 3", control)
|
||||
self.assertIn("$Payload.performance_mode", control)
|
||||
self.assertIn("physical_control_disabled_stage_a", control)
|
||||
self.assertNotIn('"set.miner.', control)
|
||||
self.assertNotIn("$Payload.performance_mode", control)
|
||||
self.assertNotIn("api3_account =", control)
|
||||
self.assertNotIn("api3_password =", control)
|
||||
for forbidden in ("raw_response", "raw_command", "raw_exception"):
|
||||
@@ -366,3 +379,18 @@ class AgentEnrichmentIsolationSourceTests(unittest.TestCase):
|
||||
self.assertIn("minerops_require_agent_secret(request)", route)
|
||||
self.assertIn('"site_name": "Plexus"', route)
|
||||
self.assertNotIn("Depends(require_agent_identity)", route)
|
||||
|
||||
def test_disabled_old_command_routes_are_fixed_retirement_responses(self):
|
||||
main = (Path(__file__).parents[1] / "app" / "main.py").read_text()
|
||||
for decorator in (
|
||||
'@app.get("/api/agent/commands/pending-disabled-old")',
|
||||
'@app.post("/api/agent/commands/result-disabled-old")',
|
||||
):
|
||||
with self.subTest(decorator=decorator):
|
||||
start = main.index(decorator)
|
||||
end = main.index("\n@app.", start + 1)
|
||||
route = main[start:end]
|
||||
self.assertIn("status_code=410", route)
|
||||
self.assertNotIn("minerops_require_agent_secret", route)
|
||||
self.assertNotIn("request.json", route)
|
||||
self.assertNotIn("minerops_agent_command_queue", route)
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import unittest
|
||||
from uuid import UUID
|
||||
from unittest.mock import patch
|
||||
|
||||
import psycopg
|
||||
from psycopg.errors import ForeignKeyViolation, UniqueViolation
|
||||
@@ -10,6 +11,8 @@ from app.agent_zones.repository import AgentZoneRepository
|
||||
from app.config import settings
|
||||
from app.miner_control.repository import MinerControlRepository
|
||||
from app.miner_control.service import MinerControlService
|
||||
from app.miner_control.exceptions import MinerControlUnavailable
|
||||
from app.agent_commands.policy import PHYSICAL_CONTROL_DISABLED
|
||||
|
||||
|
||||
SITE_A = UUID("00000000-0000-4000-8000-000000001630")
|
||||
@@ -34,6 +37,29 @@ def connect():
|
||||
|
||||
|
||||
class AgentZonePostgresTests(unittest.TestCase):
|
||||
def assert_stage_a_unavailable(self, expected_zone, expected_agent):
|
||||
repository = MinerControlRepository(connect)
|
||||
context = repository.get_control_context(MINER_UNASSIGNED)
|
||||
# Check actual routing facts independently of the unconditional Stage A
|
||||
# reason. A changed assertion must not hide a Site fallback regression.
|
||||
self.assertEqual(context["agent_zone_id"], expected_zone)
|
||||
self.assertEqual(context["agent_id"], expected_agent)
|
||||
projection = MinerControlService(repository).get_controls(MINER_UNASSIGNED)
|
||||
for capability in (projection.service_control, projection.performance_mode_control):
|
||||
self.assertEqual(capability.state, "unavailable")
|
||||
self.assertEqual(capability.reason, PHYSICAL_CONTROL_DISABLED)
|
||||
self.assertIsNone(projection.available_service_action)
|
||||
with patch.object(repository, "enqueue_with_audit") as enqueue:
|
||||
for action, value in (("suspend", None), ("resume", None), ("performance_mode", "high")):
|
||||
with self.assertRaisesRegex(MinerControlUnavailable, PHYSICAL_CONTROL_DISABLED):
|
||||
MinerControlService(repository).queue_control(
|
||||
MINER_UNASSIGNED, action, requested_value=value,
|
||||
actor_user_id=UUID("00000000-0000-4000-8000-000000001639"),
|
||||
actor_username="zone-containment-test",
|
||||
)
|
||||
enqueue.assert_not_called()
|
||||
return context
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
connection = connect()
|
||||
@@ -203,10 +229,8 @@ class AgentZonePostgresTests(unittest.TestCase):
|
||||
finally:
|
||||
connection.close()
|
||||
try:
|
||||
projection = MinerControlService(MinerControlRepository(connect)).get_controls(
|
||||
MINER_UNASSIGNED
|
||||
)
|
||||
self.assertEqual(projection.service_control.reason, "agent_zone_disabled")
|
||||
context = self.assert_stage_a_unavailable(ZONE_A, AGENT_A)
|
||||
self.assertFalse(context["agent_zone_enabled"])
|
||||
finally:
|
||||
connection = connect()
|
||||
try:
|
||||
@@ -228,16 +252,10 @@ class AgentZonePostgresTests(unittest.TestCase):
|
||||
connection.commit()
|
||||
finally:
|
||||
connection.close()
|
||||
projection = MinerControlService(MinerControlRepository(connect)).get_controls(
|
||||
MINER_UNASSIGNED
|
||||
)
|
||||
self.assertEqual(projection.service_control.reason, "agent_unavailable")
|
||||
self.assert_stage_a_unavailable(ZONE_C, None)
|
||||
|
||||
def test_null_zone_does_not_fall_back_to_site_agent(self):
|
||||
projection = MinerControlService(MinerControlRepository(connect)).get_controls(
|
||||
MINER_UNASSIGNED
|
||||
)
|
||||
self.assertEqual(projection.service_control.reason, "agent_zone_unassigned")
|
||||
self.assert_stage_a_unavailable(None, None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Focused regression tests for retired legacy miner-write surfaces."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import main
|
||||
from app.identity import get_current_user, require_permission, required_api_permission
|
||||
|
||||
|
||||
APP_ROOT = Path(__file__).parents[1] / "app"
|
||||
|
||||
|
||||
class LegacyControlRetirementTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def assert_retired(callable_, *args, **kwargs):
|
||||
try:
|
||||
result = callable_(*args, **kwargs)
|
||||
if inspect.isawaitable(result):
|
||||
asyncio.run(result)
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 410:
|
||||
raise AssertionError(f"Expected HTTP 410, got {exc.status_code}") from exc
|
||||
return
|
||||
raise AssertionError("Expected a fixed HTTP 410 retirement response")
|
||||
|
||||
def test_all_legacy_http_write_routes_are_fixed_inert_responses(self):
|
||||
routes = (
|
||||
(main.run_dashboard_m30_command_test_lab, {"role": "Admin"}),
|
||||
(main.retired_direct_reboot_miner_command, "miner-1", {"role": "Admin"}),
|
||||
(main.real_power_off_miner_command, "miner-1", {"role": "Admin"}),
|
||||
(main.real_power_on_miner_command, "miner-1", {"role": "Admin"}),
|
||||
(main.real_suspend_miner_command, "miner-1", {"role": "Admin"}),
|
||||
(main.real_resume_miner_command, "miner-1", {"role": "Admin"}),
|
||||
(main.retired_direct_api3_password_change, "miner-1", {"role": "Admin"}),
|
||||
(main.retired_legacy_agent_miner_control, "miner-1", {"role": "Admin"}),
|
||||
(main.process_due_dashboard_scheduled_bulk_actions, {"role": "Admin"}),
|
||||
(main.create_dashboard_scheduled_bulk_action, {"role": "Admin"}),
|
||||
(main.whatsminer_read_test, {"role": "Admin"}),
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(main, "get_db_connection") as sync_database,
|
||||
patch.object(main, "minerops_command_db_connect") as async_database,
|
||||
patch.object(main, "insert_miner_command_event") as audit_write,
|
||||
patch.object(main, "minerops_send_whatsminer_reboot") as reboot_write,
|
||||
patch.object(main, "minerops_power_command") as power_write,
|
||||
patch.object(main, "minerops_api3_change_password") as password_write,
|
||||
):
|
||||
for route in routes:
|
||||
with self.subTest(route=route[0].__name__):
|
||||
self.assert_retired(route[0], *route[1:])
|
||||
|
||||
for forbidden_call in (
|
||||
sync_database,
|
||||
async_database,
|
||||
audit_write,
|
||||
reboot_write,
|
||||
power_write,
|
||||
password_write,
|
||||
):
|
||||
self.assertFalse(forbidden_call.called)
|
||||
|
||||
def test_m30_requires_control_permission_before_inert_response(self):
|
||||
self.assertEqual(
|
||||
required_api_permission(
|
||||
"POST", "/api/dashboard/m30-command-test-lab/run"
|
||||
),
|
||||
"miners.control",
|
||||
)
|
||||
dependency = require_permission("miners.control")
|
||||
with self.assertRaises(HTTPException) as denied:
|
||||
dependency({"role": "Read Only", "permissions": []})
|
||||
self.assertEqual(denied.exception.status_code, 403)
|
||||
|
||||
route_source = inspect.getsource(main.run_dashboard_m30_command_test_lab)
|
||||
self.assertIn('Depends(require_permission("miners.control"))', route_source)
|
||||
self.assertIn("status_code=410", route_source)
|
||||
self.assertNotIn("minerops_agent_command_queue", route_source)
|
||||
self.assertNotIn("plexus-windows-agent", route_source)
|
||||
self.assertNotIn("dry_run", route_source)
|
||||
|
||||
app = FastAPI()
|
||||
app.add_api_route(
|
||||
"/api/dashboard/m30-command-test-lab/run",
|
||||
main.run_dashboard_m30_command_test_lab,
|
||||
methods=["POST"],
|
||||
)
|
||||
with TestClient(app) as client:
|
||||
app.dependency_overrides[get_current_user] = lambda: {
|
||||
"role": "Read Only",
|
||||
"permissions": [],
|
||||
}
|
||||
self.assertEqual(
|
||||
client.post("/api/dashboard/m30-command-test-lab/run").status_code,
|
||||
403,
|
||||
)
|
||||
app.dependency_overrides[get_current_user] = lambda: {
|
||||
"role": "Custom",
|
||||
"permissions": ["miners.control"],
|
||||
}
|
||||
self.assertEqual(
|
||||
client.post("/api/dashboard/m30-command-test-lab/run").status_code,
|
||||
410,
|
||||
)
|
||||
|
||||
def test_registered_retired_routes_have_no_direct_write_implementation(self):
|
||||
retired_paths = {
|
||||
"/api/dashboard/m30-command-test-lab/run",
|
||||
"/api/dashboard/scheduled-bulk-actions/process-due",
|
||||
"/api/dashboard/scheduled-bulk-actions",
|
||||
"/api/tools/whatsminer-read-test",
|
||||
"/api/miners/{miner_id}/commands/reboot",
|
||||
"/api/miners/{miner_id}/commands/power-off",
|
||||
"/api/miners/{miner_id}/commands/power-on",
|
||||
"/api/miners/{miner_id}/commands/suspend",
|
||||
"/api/miners/{miner_id}/commands/resume",
|
||||
"/api/miners/{miner_id}/api3/change-password",
|
||||
"/api/miners/{miner_id}/agent-control",
|
||||
}
|
||||
registered = [
|
||||
(route.path, route.endpoint)
|
||||
for route in main.app.routes
|
||||
if getattr(route, "path", None) in retired_paths
|
||||
and "POST" in getattr(route, "methods", set())
|
||||
]
|
||||
self.assertEqual({path for path, _ in registered}, retired_paths)
|
||||
self.assertEqual(len(registered), len(retired_paths))
|
||||
|
||||
forbidden = (
|
||||
"socket",
|
||||
"minerops_send_",
|
||||
"minerops_power_command",
|
||||
"minerops_api3_change_password",
|
||||
"insert_miner_command_event",
|
||||
"minerops_agent_command_queue",
|
||||
"minerops_command_db_connect",
|
||||
"get_db_connection",
|
||||
"request.json",
|
||||
)
|
||||
for path, endpoint in registered:
|
||||
source = inspect.getsource(endpoint)
|
||||
with self.subTest(path=path):
|
||||
self.assertIn("status_code=410", source)
|
||||
for unsafe_fragment in forbidden:
|
||||
self.assertNotIn(unsafe_fragment, source)
|
||||
|
||||
def test_legacy_scheduler_is_noop_and_not_registered_before_claim(self):
|
||||
with patch.object(main, "get_db_connection") as database:
|
||||
self.assertEqual(main.minerops_process_due_scheduled_bulk_actions(), [])
|
||||
self.assertFalse(database.called)
|
||||
self.assertFalse(hasattr(main.app.state, "agent_command_before_claim"))
|
||||
source = inspect.getsource(main.minerops_process_due_scheduled_bulk_actions)
|
||||
self.assertNotIn("minerops_agent_command_queue", source)
|
||||
self.assertNotIn("get_db_connection", source)
|
||||
|
||||
def test_every_retired_http_route_denies_unprivileged_users_and_never_writes(self):
|
||||
paths = {
|
||||
"/api/dashboard/m30-command-test-lab/run",
|
||||
"/api/dashboard/scheduled-bulk-actions",
|
||||
"/api/dashboard/scheduled-bulk-actions/process-due",
|
||||
"/api/tools/whatsminer-read-test",
|
||||
"/api/miners/{miner_id}/commands/reboot",
|
||||
"/api/miners/{miner_id}/commands/power-off",
|
||||
"/api/miners/{miner_id}/commands/power-on",
|
||||
"/api/miners/{miner_id}/commands/suspend",
|
||||
"/api/miners/{miner_id}/commands/resume",
|
||||
"/api/miners/{miner_id}/api3/change-password",
|
||||
"/api/miners/{miner_id}/agent-control",
|
||||
}
|
||||
app = FastAPI()
|
||||
for route in main.app.routes:
|
||||
if getattr(route, "path", None) in paths and "POST" in route.methods:
|
||||
app.add_api_route(route.path, route.endpoint, methods=["POST"])
|
||||
with TestClient(app) as client, patch.object(main.socket, "create_connection") as transport, patch.object(main, "get_db_connection") as database:
|
||||
for permissions, expected in (([], 403), (["miners.control", "credentials.manage"], 410)):
|
||||
app.dependency_overrides[get_current_user] = lambda: {
|
||||
"role": "Custom", "permissions": permissions,
|
||||
}
|
||||
for path in paths:
|
||||
concrete = path.replace("{miner_id}", "miner-1")
|
||||
self.assertEqual(client.post(concrete, json={"dry_run": False}).status_code, expected, concrete)
|
||||
transport.assert_not_called()
|
||||
database.assert_not_called()
|
||||
|
||||
def test_diagnostic_write_disguises_overrides_and_api3_are_retired(self):
|
||||
app = FastAPI()
|
||||
app.add_api_route("/api/tools/whatsminer-read-test", main.whatsminer_read_test, methods=["POST"])
|
||||
app.dependency_overrides[get_current_user] = lambda: {"role": "Custom", "permissions": ["miners.control"]}
|
||||
attacks = (
|
||||
{"command": "reboot"}, {"command": "read_then_reboot"},
|
||||
{"command": "summary", "params": {"command": "reboot"}},
|
||||
{"command": "summary", "params": {"cmd": "set.miner.service", "param": "stop"}},
|
||||
{"port": 4433, "command": "set.miner.power_mode", "params": {"param": "high"}},
|
||||
{"command": "summary\n{\"command\":\"reboot\"}"},
|
||||
{"command": "summary"},
|
||||
)
|
||||
with TestClient(app) as client, patch.object(main.socket, "create_connection") as transport, patch.object(main, "send_whatsminer_read_command") as sender:
|
||||
for attack in attacks:
|
||||
self.assertEqual(client.post("/api/tools/whatsminer-read-test", json={"ip_address": "192.0.2.1", **attack}).status_code, 410)
|
||||
transport.assert_not_called()
|
||||
sender.assert_not_called()
|
||||
|
||||
def test_internal_discovery_helper_cannot_become_a_generic_command_gateway(self):
|
||||
with patch.object(main.socket, "create_connection") as transport:
|
||||
for command, port, params in (
|
||||
("reboot", 4028, None), ("read_reboot", 4028, None),
|
||||
("summary", 4433, None), ("set.miner.service", 4433, {"param": "stop"}),
|
||||
("summary", 4028, {"command": "reboot"}),
|
||||
("summary", 4028, {"cmd": "set.miner.service"}),
|
||||
("get_miner_info", 4028, {"info": "arbitrary"}),
|
||||
):
|
||||
self.assert_retired(main.send_whatsminer_read_command, "192.0.2.1", port, command, 500, params)
|
||||
transport.assert_not_called()
|
||||
|
||||
def test_old_agent_aliases_are_inert_even_without_authentication(self):
|
||||
app = FastAPI()
|
||||
app.add_api_route("/api/agent/commands/pending-disabled-old", main.minerops_agent_commands_pending_retired, methods=["GET"])
|
||||
app.add_api_route("/api/agent/commands/result-disabled-old", main.minerops_agent_commands_result_retired, methods=["POST"])
|
||||
with TestClient(app) as client, patch.object(main, "minerops_command_db_connect") as database:
|
||||
self.assertEqual(client.get("/api/agent/commands/pending-disabled-old").status_code, 410)
|
||||
self.assertEqual(client.post("/api/agent/commands/result-disabled-old", json={"status": "completed"}).status_code, 410)
|
||||
database.assert_not_called()
|
||||
|
||||
def test_canonical_control_still_derives_agent_from_miner_zone(self):
|
||||
repository_source = (
|
||||
APP_ROOT / "miner_control" / "repository.py"
|
||||
).read_text()
|
||||
self.assertIn("miners.agent_zone_id", repository_source)
|
||||
self.assertIn(
|
||||
"agents.agent_zone_id = miners.agent_zone_id",
|
||||
repository_source,
|
||||
)
|
||||
self.assertNotIn("agents.site_id = miners.site_id", repository_source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -71,64 +71,47 @@ class FakeRepository:
|
||||
|
||||
|
||||
class MinerControlServiceTests(unittest.TestCase):
|
||||
def test_supported_projection_is_factual_and_bounded(self):
|
||||
def test_disabled_projection_preserves_facts_and_bounded_history(self):
|
||||
repository = FakeRepository()
|
||||
result = MinerControlService(repository).get_controls(MINER_ID)
|
||||
self.assertEqual(result.service_control.state, "supported")
|
||||
self.assertEqual(result.service_control.state, "unavailable")
|
||||
self.assertEqual(result.service_control.reason, "physical_control_disabled_stage_a")
|
||||
self.assertEqual(result.performance_mode, "normal")
|
||||
self.assertEqual(result.available_service_action, "suspend")
|
||||
self.assertIsNone(result.available_service_action)
|
||||
self.assertEqual(repository.limits, [5])
|
||||
|
||||
def test_capability_missing_facts_is_not_optimistic(self):
|
||||
cases = (
|
||||
({"agent_zone_id": None}, "unavailable", "agent_zone_unassigned"),
|
||||
({"agent_zone_exists": False}, "unavailable", "agent_zone_missing"),
|
||||
({"agent_zone_enabled": False}, "unavailable", "agent_zone_disabled"),
|
||||
({"agent_id": None}, "unavailable", "agent_unavailable"),
|
||||
({"current_ip": None}, "unavailable", "current_ip_missing"),
|
||||
({"api3_supported": False}, "unavailable", "api3_unsupported"),
|
||||
({"api3_supported": None}, "unknown", "api3_support_unknown"),
|
||||
({"last_payload": {}}, "unknown", "agent_capability_unknown"),
|
||||
(
|
||||
{"last_payload": {"control_available": False,
|
||||
"control_unavailable_reason": "credentials_missing"}},
|
||||
"unavailable", "credentials_missing",
|
||||
),
|
||||
)
|
||||
for changes, state, reason in cases:
|
||||
with self.subTest(reason=reason):
|
||||
result = MinerControlService(FakeRepository(context(**changes))).get_controls(MINER_ID)
|
||||
self.assertEqual(result.service_control.state, state)
|
||||
self.assertEqual(result.service_control.reason, reason)
|
||||
def test_capability_cannot_be_enabled_by_agent_claims(self):
|
||||
for facts in ({}, {"control_available": True}, {"control_available": False}):
|
||||
repository = FakeRepository(context(last_payload=facts))
|
||||
result = MinerControlService(repository).get_controls(MINER_ID)
|
||||
self.assertEqual(result.service_control.state, "unavailable")
|
||||
self.assertEqual(result.service_control.reason, "physical_control_disabled_stage_a")
|
||||
|
||||
def test_resume_requires_known_stopped_state(self):
|
||||
service = MinerControlService(FakeRepository(context(runtime_state="offline")))
|
||||
with self.assertRaisesRegex(MinerControlUnavailable, "factually_stopped"):
|
||||
service.queue_control(
|
||||
MINER_ID, "resume", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="operator",
|
||||
)
|
||||
def test_resume_is_disabled_even_for_known_stopped_state(self):
|
||||
service = MinerControlService(FakeRepository(context(runtime_state="stopped")))
|
||||
with self.assertRaisesRegex(MinerControlUnavailable, "physical_control_disabled_stage_a"):
|
||||
service.queue_control(MINER_ID, "resume", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="operator")
|
||||
|
||||
def test_queue_contains_routing_but_audit_and_response_do_not(self):
|
||||
def test_all_current_actions_are_rejected_before_routing_or_queue_write(self):
|
||||
for action in ("suspend", "resume", "performance_mode"):
|
||||
repository = FakeRepository()
|
||||
with self.assertRaisesRegex(MinerControlUnavailable, "physical_control_disabled_stage_a"):
|
||||
MinerControlService(repository).queue_control(
|
||||
MINER_ID, action, requested_value="high",
|
||||
actor_user_id=USER_ID, actor_username="operator")
|
||||
self.assertEqual(repository.enqueue_calls, [])
|
||||
self.assertFalse(hasattr(repository, "requested_miner"))
|
||||
|
||||
def test_rejected_request_leaves_context_unchanged(self):
|
||||
repository = FakeRepository()
|
||||
result = MinerControlService(repository).queue_control(
|
||||
MINER_ID, "performance_mode", requested_value="high",
|
||||
actor_user_id=USER_ID, actor_username="operator",
|
||||
)
|
||||
call = repository.enqueue_calls[0]
|
||||
self.assertEqual(call["request_payload"]["ip_address"], "192.0.2.10")
|
||||
self.assertNotIn("ip_address", call["audit_payload"])
|
||||
self.assertNotIn("agent_id", call["audit_payload"])
|
||||
self.assertNotIn("password", str(call).lower())
|
||||
self.assertNotIn("192.0.2.10", result.model_dump_json())
|
||||
|
||||
def test_no_projection_changes_when_work_is_queued(self):
|
||||
repository = FakeRepository()
|
||||
MinerControlService(repository).queue_control(
|
||||
MINER_ID, "suspend", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="operator",
|
||||
)
|
||||
self.assertFalse(any("update" in key for key in repository.enqueue_calls[0]))
|
||||
before = repository.miner_context.copy()
|
||||
with self.assertRaises(MinerControlUnavailable):
|
||||
MinerControlService(repository).queue_control(
|
||||
MINER_ID, "suspend", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="operator")
|
||||
self.assertEqual(repository.miner_context, before)
|
||||
self.assertEqual(repository.enqueue_calls, [])
|
||||
|
||||
|
||||
class MinerControlRouterTests(unittest.TestCase):
|
||||
@@ -159,18 +142,16 @@ class MinerControlRouterTests(unittest.TestCase):
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.json()["data"]["miner_id"], str(MINER_ID))
|
||||
|
||||
def test_posts_require_control_and_return_202_bounded_envelope(self):
|
||||
path = f"/api/miners/{MINER_ID}/controls/suspend"
|
||||
self.authenticate(("miners.view",))
|
||||
self.assertEqual(self.client.post(path, json={}).status_code, 403)
|
||||
self.authenticate(("miners.control",))
|
||||
response = self.client.post(path, json={})
|
||||
self.assertEqual(response.status_code, 202)
|
||||
self.assertEqual(set(response.json()["data"]), {
|
||||
"operation_id", "miner_id", "action", "requested_value", "status", "created_at"
|
||||
})
|
||||
self.assertNotIn("ip", response.text.lower())
|
||||
self.assertNotIn("agent", response.text.lower())
|
||||
def test_posts_require_control_and_then_fail_closed(self):
|
||||
for action, body in (("suspend", {}), ("resume", {}), ("performance-mode", {"mode": "high"})):
|
||||
path = f"/api/miners/{MINER_ID}/controls/{action}"
|
||||
self.authenticate(("miners.view",))
|
||||
self.assertEqual(self.client.post(path, json=body).status_code, 403)
|
||||
self.authenticate(("miners.control",))
|
||||
response = self.client.post(path, json=body)
|
||||
self.assertEqual(response.status_code, 409)
|
||||
self.assertIn("physical_control_disabled_stage_a", response.text)
|
||||
self.assertEqual(self.repository.enqueue_calls, [])
|
||||
|
||||
def test_strict_bodies_reject_routing_commands_and_credentials(self):
|
||||
self.authenticate(("miners.control",))
|
||||
@@ -195,7 +176,7 @@ class MinerControlRouterTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(
|
||||
self.client.post(path, json={}, headers={"Origin": "http://testserver"}).status_code,
|
||||
202,
|
||||
409,
|
||||
)
|
||||
|
||||
def test_identity_permission_mapping_is_specific(self):
|
||||
|
||||
@@ -137,7 +137,7 @@ class MinerControlPostgresTests(unittest.TestCase):
|
||||
"""
|
||||
INSERT INTO minerops_agent_command_queue (
|
||||
id, agent_id, command_type, command_status, request_payload, created_at
|
||||
) VALUES (%s, %s, 'miner_control', %s, %s, COALESCE(%s, now()))
|
||||
) VALUES (%s, %s, 'poll_known_miners_now', %s, %s, COALESCE(%s, now()))
|
||||
""",
|
||||
(command_id, AGENT_ID, status,
|
||||
Jsonb({"schema_version": 1, "miner_id": str(miner_id),
|
||||
@@ -199,13 +199,22 @@ class MinerControlPostgresTests(unittest.TestCase):
|
||||
service.submit_result(AGENT_ID, pending)
|
||||
|
||||
def test_atomic_queue_audit_duplicate_live_and_completed_history(self):
|
||||
service = MinerControlService(MinerControlRepository(connect))
|
||||
first = service.queue_control(
|
||||
repository = MinerControlRepository(connect)
|
||||
def queue_control(miner_id, action, *, requested_value, actor_user_id, actor_username):
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
return SimpleNamespace(**repository.enqueue_with_audit(
|
||||
operation_id=str(uuid4()), miner_id=miner_id, agent_id=AGENT_ID,
|
||||
actor_user_id=actor_user_id, actor_username=actor_username,
|
||||
request_payload={"schema_version": 1, "miner_id": str(miner_id),
|
||||
"action": action, "requested_value": requested_value},
|
||||
audit_payload={"miner_id": str(miner_id), "action": action}))
|
||||
first = queue_control(
|
||||
MINER_ID, "suspend", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="control-test-user",
|
||||
)
|
||||
with self.assertRaises(MinerControlConflict):
|
||||
service.queue_control(
|
||||
queue_control(
|
||||
MINER_ID, "suspend", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="control-test-user",
|
||||
)
|
||||
@@ -223,17 +232,26 @@ class MinerControlPostgresTests(unittest.TestCase):
|
||||
connection.commit()
|
||||
finally:
|
||||
connection.close()
|
||||
second = service.queue_control(
|
||||
second = queue_control(
|
||||
MINER_ID, "suspend", requested_value=None,
|
||||
actor_user_id=USER_ID, actor_username="control-test-user",
|
||||
)
|
||||
self.assertNotEqual(first.operation_id, second.operation_id)
|
||||
|
||||
def test_audit_failure_rolls_back_queue_row(self):
|
||||
service = MinerControlService(MinerControlRepository(connect))
|
||||
repository = MinerControlRepository(connect)
|
||||
def queue_control(miner_id, action, *, requested_value, actor_user_id, actor_username):
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
return SimpleNamespace(**repository.enqueue_with_audit(
|
||||
operation_id=str(uuid4()), miner_id=miner_id, agent_id=AGENT_ID,
|
||||
actor_user_id=actor_user_id, actor_username=actor_username,
|
||||
request_payload={"schema_version": 1, "miner_id": str(miner_id),
|
||||
"action": action, "requested_value": requested_value},
|
||||
audit_payload={"miner_id": str(miner_id), "action": action}))
|
||||
operation_count_before = self._operation_count()
|
||||
with self.assertRaises(Exception):
|
||||
service.queue_control(
|
||||
queue_control(
|
||||
MINER_ID, "suspend", requested_value=None,
|
||||
actor_user_id=UUID("00000000-0000-4000-8000-000000009999"),
|
||||
actor_username="missing-user",
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Defense-in-depth unit tests; no physical miner or external service is used."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app import main
|
||||
from app.agent_commands.exceptions import AgentCommandConflict
|
||||
from app.agent_commands.models import AgentCommandResultRequest
|
||||
from app.agent_commands.policy import rejection_reason
|
||||
from app.agent_commands.service import AgentCommandService
|
||||
|
||||
|
||||
class StageAContainmentTests(unittest.TestCase):
|
||||
def test_service_dispatch_rejects_physical_and_unknown_rows_even_if_repository_regresses(self):
|
||||
repository = Mock()
|
||||
repository.claim_pending.return_value = [
|
||||
{"command_id": str(index), "command_type": kind, "request_payload": payload,
|
||||
"created_at": datetime.now(timezone.utc)}
|
||||
for index, (kind, payload) in enumerate((
|
||||
("miner_control", {"schema_version": 1, "action": "suspend"}),
|
||||
("miner_control", {"schema_version": 999}),
|
||||
("miner_control_v2", {}), ("raw_api3", {}),
|
||||
("poll_known_miners_now", {}),
|
||||
))
|
||||
]
|
||||
result = AgentCommandService(repository).claim("simulated-agent")
|
||||
self.assertEqual([row.command_type for row in result], ["poll_known_miners_now"])
|
||||
|
||||
def test_service_result_rejection_does_not_depend_on_payload_validation(self):
|
||||
for kind in ("miner_control", "raw_api3", "miner_control_v2"):
|
||||
repository = Mock()
|
||||
repository.get_command.return_value = {"id": "old-command", "agent_id": "simulated-agent",
|
||||
"command_type": kind, "command_status": "claimed", "request_payload": {}}
|
||||
with self.assertRaises(AgentCommandConflict):
|
||||
AgentCommandService(repository).submit_result("simulated-agent",
|
||||
AgentCommandResultRequest(command_id="old-command", status="completed",
|
||||
result_payload={"schema_version": 999, "command_accepted": True}))
|
||||
args = repository.finish_command.call_args.args
|
||||
self.assertEqual(args[2:4], ("cancelled", {}))
|
||||
self.assertEqual(args[4], rejection_reason(kind))
|
||||
|
||||
def test_dormant_legacy_helpers_cannot_transmit_or_queue_if_called_internally(self):
|
||||
names = (
|
||||
"dry_run_reboot_miner_command", "change_miner_api3_password",
|
||||
"minerops_queue_agent_miner_control", "minerops_power_command",
|
||||
"minerops_send_whatsminer_reboot_sync", "minerops_send_whatsminer_reboot",
|
||||
"minerops_send_whatsminer_command_sync", "minerops_api3_send_payload_sync",
|
||||
"minerops_api3_service_sync", "minerops_api3_service",
|
||||
"minerops_send_control_command_item", "minerops_send_whatsminer_command_sequence",
|
||||
"minerops_send_whatsminer_command", "minerops_api3_service_sync_with_credentials",
|
||||
"minerops_api3_service_with_credentials", "minerops_api3_change_password_sync",
|
||||
"minerops_api3_change_password",
|
||||
)
|
||||
with patch.object(main.socket, "create_connection") as transport, patch.object(main, "get_db_connection") as database, patch.object(main, "minerops_command_db_connect") as async_database:
|
||||
for name in names:
|
||||
function = getattr(main, name)
|
||||
required = ["simulated" for parameter in inspect.signature(function).parameters.values()
|
||||
if parameter.default is inspect.Parameter.empty]
|
||||
with self.subTest(name=name), self.assertRaises(HTTPException) as rejected:
|
||||
result = function(*required)
|
||||
if inspect.isawaitable(result):
|
||||
asyncio.run(result)
|
||||
self.assertEqual(rejected.exception.status_code, 410)
|
||||
transport.assert_not_called()
|
||||
database.assert_not_called()
|
||||
async_database.assert_not_called()
|
||||
|
||||
def test_fixed_internal_discovery_queries_still_use_only_allowlisted_selection(self):
|
||||
parameters = {"get_miner_info": {"info": "ip,proto,netmask,gateway,dns,hostname,mac,ledstat,minersn,powersn"},
|
||||
"get_version": {}, "summary": {}, "edevs": {}, "pools": {}}
|
||||
for command, params in parameters.items():
|
||||
with patch.object(main.socket, "create_connection") as transport:
|
||||
socket = transport.return_value.__enter__.return_value
|
||||
socket.recv.side_effect = [b'{"STATUS":[]}', b'']
|
||||
result = main.send_whatsminer_read_command("192.0.2.1", 4028, command, 500, params)
|
||||
self.assertTrue(result["success"])
|
||||
self.assertEqual(socket.sendall.call_count, 1)
|
||||
import json
|
||||
transmitted = json.loads(socket.sendall.call_args.args[0])
|
||||
self.assertEqual(transmitted, {"command": command, **params})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Stage A queue containment; requires an explicitly disposable PostgreSQL DB."""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from app.agent_commands.exceptions import AgentCommandConflict
|
||||
from app.agent_commands.models import AgentCommandResultRequest
|
||||
from app.agent_commands.repository import AgentCommandRepository
|
||||
from app.agent_commands.service import AgentCommandService
|
||||
from app.config import settings
|
||||
|
||||
|
||||
AGENT = "stage-a-simulated-agent"
|
||||
OTHER_AGENT = "stage-a-other-agent"
|
||||
|
||||
|
||||
def connect():
|
||||
return psycopg.connect(host=settings.postgres_host, port=settings.postgres_port,
|
||||
dbname=settings.postgres_db, user=settings.postgres_user,
|
||||
password=settings.postgres_password, connect_timeout=5)
|
||||
|
||||
|
||||
class StageAQueueContainmentTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if os.environ.get("MINEROPS_STAGE_A_DISPOSABLE") != "1" or not settings.postgres_db.startswith("minerops_stage_a_"):
|
||||
raise RuntimeError("Refusing queue tests outside an explicitly disposable Stage A database")
|
||||
|
||||
def setUp(self):
|
||||
self.transport_patch = patch("socket.create_connection", side_effect=AssertionError("No miner transport allowed"))
|
||||
self.transport = self.transport_patch.start()
|
||||
self.addCleanup(self.transport_patch.stop)
|
||||
self.addCleanup(self.transport.assert_not_called)
|
||||
with connect() as connection, connection.cursor() as cur:
|
||||
cur.execute("DELETE FROM audit_logs WHERE entity_type = 'agent_command' AND old_value_json ->> 'agent_id' IN (%s, %s)", (AGENT, OTHER_AGENT))
|
||||
cur.execute("DELETE FROM minerops_agent_command_queue WHERE agent_id IN (%s, %s)", (AGENT, OTHER_AGENT))
|
||||
self.repository = AgentCommandRepository(connect)
|
||||
self.service = AgentCommandService(self.repository)
|
||||
|
||||
@staticmethod
|
||||
def seed(command_id, status="pending", command_type="miner_control", payload=None, agent=AGENT):
|
||||
if payload is None:
|
||||
payload = {"action": "legacy_4028", "ip_address": "192.0.2.1", "legacy_4028_command": "reboot", "dry_run": False}
|
||||
with connect() as connection, connection.cursor() as cur:
|
||||
cur.execute("""INSERT INTO minerops_agent_command_queue
|
||||
(id, agent_id, command_type, command_status, request_payload,
|
||||
result_payload, error_message, claimed_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, 'prior evidence',
|
||||
CASE WHEN %s = 'claimed' THEN now() ELSE NULL END)""",
|
||||
(command_id, agent, command_type, status, Jsonb(payload),
|
||||
Jsonb({"historical_evidence": True}), status))
|
||||
|
||||
@staticmethod
|
||||
def row(command_id):
|
||||
with connect() as connection, connection.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT * FROM minerops_agent_command_queue WHERE id = %s", (command_id,))
|
||||
return dict(cur.fetchone())
|
||||
|
||||
@staticmethod
|
||||
def audits(command_id):
|
||||
with connect() as connection, connection.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT * FROM audit_logs WHERE action = 'agent.command.rejected' AND old_value_json ->> 'command_id' = %s", (command_id,))
|
||||
return [dict(row) for row in cur.fetchall()]
|
||||
|
||||
def assert_cancelled_preserving_history(self, command_id, before, outcome):
|
||||
after = self.row(command_id)
|
||||
self.assertEqual(after["command_status"], "cancelled")
|
||||
for field in ("id", "agent_id", "command_type", "request_payload", "result_payload", "created_at", "claimed_at"):
|
||||
self.assertEqual(after[field], before[field], field)
|
||||
self.assertTrue(after["error_message"].startswith("prior evidence\n"))
|
||||
self.assertIsNotNone(after["completed_at"])
|
||||
audits = self.audits(command_id)
|
||||
self.assertEqual(len(audits), 1)
|
||||
self.assertEqual(audits[0]["old_value_json"]["status"], before["command_status"])
|
||||
self.assertEqual(audits[0]["new_value_json"]["execution_outcome"], outcome)
|
||||
self.assertNotIn("192.0.2.1", str(audits))
|
||||
|
||||
def test_seeded_legacy_pending_never_dispatched(self):
|
||||
self.seed("stage-a-pending")
|
||||
before = self.row("stage-a-pending")
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
self.assert_cancelled_preserving_history("stage-a-pending", before, "unknown")
|
||||
|
||||
def test_seeded_legacy_claimed_cancelled_with_unknown_prior_execution(self):
|
||||
self.seed("stage-a-claimed", status="claimed")
|
||||
before = self.row("stage-a-claimed")
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
self.assert_cancelled_preserving_history("stage-a-claimed", before, "unknown")
|
||||
|
||||
def test_all_physical_schemas_and_unknown_types_are_rejected(self):
|
||||
payloads = ({}, {"schema_version": 1, "action": "suspend"},
|
||||
{"schema_version": 2, "action": "resume"},
|
||||
{"schema_version": "1", "action": "performance_mode"},
|
||||
{"schema_version": 999, "action": "reboot"})
|
||||
for index, payload in enumerate(payloads):
|
||||
self.seed(f"stage-a-schema-{index}", payload=payload)
|
||||
self.seed("stage-a-unknown", command_type="raw_api3")
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
for index in range(len(payloads)):
|
||||
self.assertIn("physical_control_disabled_stage_a", self.row(f"stage-a-schema-{index}")["error_message"])
|
||||
self.assertIn("unsupported_command_schema_stage_a", self.row("stage-a-unknown")["error_message"])
|
||||
|
||||
def test_late_results_and_replay_cannot_promote_old_claimed_commands(self):
|
||||
self.seed("stage-a-late", status="claimed")
|
||||
before = self.row("stage-a-late")
|
||||
for status in ("completed", "failed", "completed"):
|
||||
request = AgentCommandResultRequest(command_id="stage-a-late", status=status,
|
||||
result_payload={"command_accepted": True})
|
||||
with self.assertRaisesRegex(AgentCommandConflict, "physical_control_disabled_stage_a"):
|
||||
self.service.submit_result(AGENT, request)
|
||||
self.assert_cancelled_preserving_history("stage-a-late", before, "unknown")
|
||||
|
||||
def test_repository_cannot_be_bypassed_by_direct_finish_call(self):
|
||||
self.seed("stage-a-direct", status="pending")
|
||||
before = self.row("stage-a-direct")
|
||||
with self.assertRaises(AgentCommandConflict):
|
||||
self.repository.finish_command("stage-a-direct", AGENT, "completed", {"accepted": True}, None)
|
||||
self.assert_cancelled_preserving_history("stage-a-direct", before, "unknown")
|
||||
|
||||
def test_wrong_agent_cannot_cancel_or_promote_another_agents_entry(self):
|
||||
self.seed("stage-a-foreign", status="claimed", agent=OTHER_AGENT)
|
||||
before = self.row("stage-a-foreign")
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
with self.assertRaises(AgentCommandConflict):
|
||||
self.service.submit_result(AGENT, AgentCommandResultRequest(command_id="stage-a-foreign", status="completed"))
|
||||
with self.assertRaises(AgentCommandConflict):
|
||||
self.repository.finish_command("stage-a-foreign", AGENT, "completed", {}, None)
|
||||
self.assertEqual(self.row("stage-a-foreign"), before)
|
||||
self.assertEqual(self.audits("stage-a-foreign"), [])
|
||||
|
||||
def test_terminal_history_is_immutable_and_not_reaudited(self):
|
||||
for status in ("completed", "failed", "cancelled"):
|
||||
command_id = f"stage-a-terminal-{status}"
|
||||
self.seed(command_id, status=status)
|
||||
before = self.row(command_id)
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
with self.assertRaises(AgentCommandConflict):
|
||||
self.repository.finish_command(command_id, AGENT, "completed", {}, None)
|
||||
self.assertEqual(self.row(command_id), before)
|
||||
self.assertEqual(self.audits(command_id), [])
|
||||
|
||||
def test_read_only_commands_still_claim_and_complete(self):
|
||||
for kind in ("poll_known_miners_now", "discover_new_miners", "enrich_match_discovery"):
|
||||
self.seed(f"stage-a-read-{kind}", command_type=kind, payload={"ip_address": "192.0.2.1"})
|
||||
commands = self.service.claim(AGENT)
|
||||
self.assertEqual(len(commands), 3)
|
||||
for command in commands:
|
||||
self.service.submit_result(AGENT, AgentCommandResultRequest(command_id=command.command_id, status="completed", result_payload={"simulated": True}))
|
||||
self.assertEqual(self.row(command.command_id)["command_status"], "completed")
|
||||
|
||||
def test_concurrent_pollers_cancel_once_and_never_deliver_physical_commands(self):
|
||||
self.seed("stage-a-concurrent")
|
||||
before = self.row("stage-a-concurrent")
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
results = list(pool.map(lambda _: AgentCommandRepository(connect).claim_pending(AGENT), range(2)))
|
||||
self.assertEqual(results, [[], []])
|
||||
self.assert_cancelled_preserving_history("stage-a-concurrent", before, "unknown")
|
||||
|
||||
def test_locked_legacy_entry_cannot_leak_into_claim_query(self):
|
||||
self.seed("stage-a-locked")
|
||||
locker = connect()
|
||||
try:
|
||||
with locker.cursor() as cur:
|
||||
cur.execute("SELECT id FROM minerops_agent_command_queue WHERE id = 'stage-a-locked' FOR UPDATE")
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
finally:
|
||||
locker.rollback()
|
||||
locker.close()
|
||||
self.assertEqual(self.service.claim(AGENT), [])
|
||||
self.assertEqual(self.row("stage-a-locked")["command_status"], "cancelled")
|
||||
|
||||
def test_audit_failure_rolls_back_cancellation_and_prevents_dispatch(self):
|
||||
self.seed("stage-a-audit-failure")
|
||||
before = self.row("stage-a-audit-failure")
|
||||
def fail_audit(cur, row, **kwargs):
|
||||
cur.execute("SELECT 1 / 0")
|
||||
with patch.object(AgentCommandRepository, "_audit_rejection", side_effect=fail_audit):
|
||||
with self.assertRaises(psycopg.errors.DivisionByZero):
|
||||
self.service.claim(AGENT)
|
||||
self.assertEqual(self.row("stage-a-audit-failure"), before)
|
||||
self.assertEqual(self.audits("stage-a-audit-failure"), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Conservative delivery outcome decisions, independent of SQL row shape."""
|
||||
import unittest
|
||||
from app.agent_commands.policy import rejection_execution_outcome
|
||||
|
||||
|
||||
class DeliveryOutcomeTests(unittest.TestCase):
|
||||
def test_only_explicit_clean_pending_evidence_allows_not_dispatched(self):
|
||||
self.assertEqual(rejection_execution_outcome({"previous_status": "pending", "no_delivery_evidence": True}), "not_dispatched")
|
||||
|
||||
def test_missing_ambiguous_or_reported_delivery_is_unknown(self):
|
||||
for row in ({}, {"previous_status": "pending"},
|
||||
{"previous_status": "pending", "no_delivery_evidence": False},
|
||||
{"previous_status": "pending", "no_delivery_evidence": None},
|
||||
{"previous_status": "pending", "no_delivery_evidence": 1},
|
||||
{"previous_status": "pending", "no_delivery_evidence": "true"},
|
||||
{"previous_status": "claimed", "no_delivery_evidence": True},
|
||||
{"previous_status": "cancelled", "no_delivery_evidence": True},
|
||||
{"previous_status": "pending", "no_delivery_evidence": True, "delivery_reported": True},
|
||||
{"previous_status": "pending", "no_delivery_evidence": True, "delivery_reported": None}):
|
||||
with self.subTest(row=row):
|
||||
self.assertEqual(rejection_execution_outcome(row), "unknown")
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Delivery evidence across every Stage A retirement entry point; no miner IO."""
|
||||
import os
|
||||
import unittest
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from app.agent_commands.exceptions import AgentCommandConflict
|
||||
from app.agent_commands.repository import AgentCommandRepository
|
||||
from app.config import settings
|
||||
from tests.test_stage_a_containment_postgres import connect
|
||||
|
||||
AGENT = "stage-a-final-evidence-agent"
|
||||
ACTOR = "00000000-0000-4000-8000-000000003302"
|
||||
SCHEDULE = "stage-a-final-evidence-schedule"
|
||||
PATHS = ("poll", "result", "cutover", "cancel", "delete")
|
||||
# Verbatim CLAIM_COMMANDS_SQL from committed 2d89e80. Test-only legacy delivery;
|
||||
# never use this query in the Stage A production repository.
|
||||
BASELINE_CLAIM_SQL = """
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM minerops_agent_command_queue
|
||||
WHERE agent_id = %s
|
||||
AND command_status = 'pending'
|
||||
ORDER BY created_at, id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT %s
|
||||
)
|
||||
UPDATE minerops_agent_command_queue AS queue
|
||||
SET command_status = 'claimed', claimed_at = now()
|
||||
FROM candidates
|
||||
WHERE queue.id = candidates.id
|
||||
AND queue.command_status = 'pending'
|
||||
RETURNING queue.id AS command_id, queue.command_type,
|
||||
queue.request_payload, queue.created_at
|
||||
"""
|
||||
|
||||
|
||||
class DeliveryEvidencePostgresTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if os.environ.get("MINEROPS_STAGE_A_DISPOSABLE") != "1" or not settings.postgres_db.startswith("minerops_stage_a_"):
|
||||
raise RuntimeError("Delivery evidence tests require a disposable Stage A database")
|
||||
|
||||
def setUp(self):
|
||||
self.repo = AgentCommandRepository(connect)
|
||||
self.clean()
|
||||
self.addCleanup(self.clean)
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO users (id,username,display_name,password_hash,role_id)
|
||||
SELECT %s,'stage-a-final-evidence','Delivery Evidence','unused',id
|
||||
FROM roles WHERE name='Admin' ON CONFLICT (id) DO NOTHING""", (ACTOR,))
|
||||
self.no_transport = patch("socket.create_connection", side_effect=AssertionError("No miner IO"))
|
||||
mocked = self.no_transport.start()
|
||||
self.addCleanup(mocked.assert_not_called)
|
||||
self.addCleanup(self.no_transport.stop)
|
||||
|
||||
def clean(self):
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM audit_logs WHERE actor_user_id=%s OR old_value_json->>'agent_id'=%s", (ACTOR, AGENT))
|
||||
cur.execute("DELETE FROM minerops_agent_command_results WHERE agent_id=%s", (AGENT,))
|
||||
cur.execute("DELETE FROM minerops_agent_command_queue WHERE agent_id=%s", (AGENT,))
|
||||
cur.execute("DELETE FROM minerops_scheduled_bulk_actions WHERE id=%s", (SCHEDULE,))
|
||||
|
||||
def seed(self, command_id="stage-a-final-evidence-command", kind="miner_control"):
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO minerops_scheduled_bulk_actions
|
||||
(id,agent_id,target_type,status,scheduled_for) VALUES (%s,%s,'all','running',now())
|
||||
ON CONFLICT (id) DO UPDATE SET status='running'""", (SCHEDULE,AGENT))
|
||||
cur.execute("""INSERT INTO minerops_agent_command_queue (id,agent_id,command_type,request_payload)
|
||||
VALUES (%s,%s,%s,%s)""",
|
||||
(command_id,AGENT,kind,Jsonb({"scheduled_action_id": SCHEDULE,"action":"suspend","ip_address":"127.0.0.1"})))
|
||||
return command_id
|
||||
|
||||
def row(self, cid):
|
||||
with connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT * FROM minerops_agent_command_queue WHERE id=%s", (cid,))
|
||||
return dict(cur.fetchone())
|
||||
|
||||
def audits(self, cid):
|
||||
with connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("""SELECT * FROM audit_logs WHERE action='agent.command.rejected'
|
||||
AND old_value_json->>'command_id'=%s""", (cid,))
|
||||
return list(cur.fetchall())
|
||||
|
||||
def retire(self, path, cid):
|
||||
if path == "poll": self.assertEqual(self.repo.claim_pending(AGENT), [])
|
||||
elif path == "result":
|
||||
with self.assertRaises(AgentCommandConflict): self.repo.finish_command(cid, AGENT, "completed", {}, None)
|
||||
elif path == "cutover": return self.repo.retire_unsupported(AGENT, actor_user_id=ACTOR)
|
||||
else: return self.repo.retire_schedule(SCHEDULE, actor_user_id=ACTOR, operation=path)
|
||||
|
||||
def assert_retired(self, cid, before, outcome):
|
||||
after = self.row(cid)
|
||||
self.assertEqual(after["command_status"], "cancelled")
|
||||
self.assertIsNotNone(after["completed_at"])
|
||||
for key in ("id","agent_id","command_type","claimed_at","created_at","request_payload","result_payload"):
|
||||
self.assertEqual(after[key], before[key], key)
|
||||
if before["error_message"]: self.assertTrue(after["error_message"].startswith(before["error_message"]+"\n"))
|
||||
audits = self.audits(cid)
|
||||
self.assertEqual(len(audits), 1)
|
||||
self.assertEqual(audits[0]["old_value_json"]["status"], before["command_status"])
|
||||
self.assertEqual(audits[0]["new_value_json"]["execution_outcome"], outcome)
|
||||
|
||||
def test_genuine_baseline_claim_then_explicit_repair_requeue_all_paths(self):
|
||||
for path in PATHS:
|
||||
with self.subTest(path=path):
|
||||
self.clean(); cid=self.seed()
|
||||
with connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(BASELINE_CLAIM_SQL,(AGENT,5)); delivered=list(cur.fetchall())
|
||||
self.assertEqual([r["command_id"] for r in delivered], [cid])
|
||||
claimed=self.row(cid)
|
||||
self.assertEqual(claimed["command_status"],"claimed"); self.assertIsNotNone(claimed["claimed_at"])
|
||||
# Explicitly simulate a legacy repair. No production requeue path
|
||||
# is asserted or added; retain the actual baseline claim evidence.
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("UPDATE minerops_agent_command_queue SET command_status='pending' WHERE id=%s",(cid,))
|
||||
before=self.row(cid); self.assertEqual(before["claimed_at"],claimed["claimed_at"])
|
||||
self.retire(path,cid); self.assert_retired(cid,before,"unknown")
|
||||
audit_before=self.audits(cid); row_before=self.row(cid)
|
||||
for retry in PATHS: self.retire(retry,cid)
|
||||
self.assertEqual(self.audits(cid),audit_before); self.assertEqual(self.row(cid),row_before)
|
||||
|
||||
def test_pristine_pending_all_paths_and_result_itself_is_delivery_evidence(self):
|
||||
for path in PATHS:
|
||||
with self.subTest(path=path):
|
||||
self.clean(); cid=self.seed(); before=self.row(cid)
|
||||
self.retire(path,cid)
|
||||
self.assert_retired(cid,before,"unknown" if path=="result" else "not_dispatched")
|
||||
|
||||
def test_each_durable_evidence_source_is_unknown_on_every_path(self):
|
||||
sources=("claim","completion","result","error","legacy_result","unlinked_legacy_result","audit_old","audit_new","prior_unknown")
|
||||
for source in sources:
|
||||
for path in PATHS:
|
||||
with self.subTest(source=source,path=path):
|
||||
self.clean(); cid=self.seed()
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
if source in ("claim","completion","result","error"):
|
||||
assignments={"claim":"claimed_at=now()", "completion":"completed_at=now()",
|
||||
"result":"result_payload='{}'::jsonb", "error":"error_message='ambiguous legacy error'"}
|
||||
cur.execute("UPDATE minerops_agent_command_queue SET "+assignments[source]+" WHERE id=%s",(cid,))
|
||||
elif source in ("legacy_result", "unlinked_legacy_result"):
|
||||
cur.execute("""INSERT INTO minerops_agent_command_results (agent_id,site_name,command_id,status,payload)
|
||||
VALUES (%s,'simulated',%s,'completed','{}'::jsonb)""",(AGENT,cid if source=="legacy_result" else None))
|
||||
else:
|
||||
# Durable audit evidence with all queue timestamps missing.
|
||||
old={"command_id":cid,"agent_id":AGENT} if source!="audit_new" else {"agent_id":AGENT}
|
||||
new={"operation_id":cid} if source=="audit_new" else {"execution_outcome":"unknown"}
|
||||
action="agent.command.rejected" if source=="prior_unknown" else "legacy.command.delivered"
|
||||
cur.execute("""INSERT INTO audit_logs (actor_user_id,action,entity_type,old_value_json,new_value_json)
|
||||
VALUES (%s,%s,'agent_command',%s,%s)""",(ACTOR,action,Jsonb(old),Jsonb(new)))
|
||||
prior_audits=self.audits(cid); before=self.row(cid); self.retire(path,cid)
|
||||
after=self.row(cid)
|
||||
self.assertEqual(after["command_status"],"cancelled")
|
||||
self.assertEqual(after["claimed_at"],before["claimed_at"])
|
||||
self.assertEqual(after["result_payload"],before["result_payload"])
|
||||
if before["completed_at"]: self.assertEqual(after["completed_at"],before["completed_at"])
|
||||
audits=self.audits(cid); self.assertEqual(len(audits),len(prior_audits)+1)
|
||||
self.assertTrue(all(a["new_value_json"].get("execution_outcome")=="unknown" for a in audits))
|
||||
|
||||
def test_creation_audit_alone_does_not_report_delivery(self):
|
||||
cid=self.seed()
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO audit_logs (actor_user_id,action,entity_type,new_value_json)
|
||||
VALUES (%s,'miner.control.queued','miner',%s)""",(ACTOR,Jsonb({"operation_id":cid})))
|
||||
before=self.row(cid); self.retire("cutover",cid); self.assert_retired(cid,before,"not_dispatched")
|
||||
|
||||
def test_currently_claimed_without_timestamp_remains_unknown(self):
|
||||
for path in PATHS:
|
||||
self.clean(); cid=self.seed()
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("UPDATE minerops_agent_command_queue SET command_status='claimed' WHERE id=%s",(cid,))
|
||||
before=self.row(cid); self.retire(path,cid); self.assert_retired(cid,before,"unknown")
|
||||
|
||||
def test_audit_failure_rolls_back_every_cancellation_path(self):
|
||||
for path in PATHS:
|
||||
with self.subTest(path=path):
|
||||
self.clean(); cid=self.seed(); before=self.row(cid)
|
||||
def fail(cur,row,**kwargs): cur.execute("SELECT 1/0")
|
||||
with patch.object(AgentCommandRepository,"_audit_rejection",side_effect=fail):
|
||||
with self.assertRaises(Exception): self.retire(path,cid)
|
||||
self.assertEqual(self.row(cid),before); self.assertEqual(self.audits(cid),[])
|
||||
|
||||
def test_concurrent_baseline_claim_blocks_result_then_unknown(self):
|
||||
cid=self.seed(); locker=connect()
|
||||
try:
|
||||
with locker.cursor() as cur:
|
||||
cur.execute(BASELINE_CLAIM_SQL,(AGENT,1)); self.assertEqual(len(cur.fetchall()),1)
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
def cancellation():
|
||||
def named_connection():
|
||||
conn=connect()
|
||||
conn.execute("SET application_name='stage-a-final-result-race'")
|
||||
return conn
|
||||
with self.assertRaises(AgentCommandConflict):
|
||||
AgentCommandRepository(named_connection).finish_command(cid,AGENT,"completed",{},None)
|
||||
future=pool.submit(cancellation)
|
||||
try:
|
||||
# Observe an actual database lock wait before committing the
|
||||
# claim, rather than relying on thread scheduling or sleeps.
|
||||
blocked=False; deadline=time.monotonic()+5
|
||||
while time.monotonic()<deadline:
|
||||
with connect() as observer, observer.cursor() as cur:
|
||||
cur.execute("""SELECT EXISTS (SELECT 1 FROM pg_stat_activity
|
||||
WHERE application_name='stage-a-final-result-race'
|
||||
AND wait_event_type='Lock')""")
|
||||
blocked=cur.fetchone()[0]
|
||||
if blocked: break
|
||||
time.sleep(.01)
|
||||
self.assertTrue(blocked,"Cancellation must wait for the genuine legacy claim lock")
|
||||
finally:
|
||||
locker.commit()
|
||||
future.result(timeout=10)
|
||||
self.assertEqual(self.row(cid)["command_status"],"cancelled")
|
||||
self.assertIsNotNone(self.row(cid)["claimed_at"])
|
||||
self.assertEqual(self.audits(cid)[0]["new_value_json"]["execution_outcome"],"unknown")
|
||||
finally: locker.close()
|
||||
|
||||
def test_mixed_legacy_backlog_classifies_each_row_without_delivering(self):
|
||||
for n in range(12):
|
||||
cid=self.seed("stage-a-final-evidence-"+str(n),"miner_control" if n%2 else "unknown_legacy")
|
||||
if n%3:
|
||||
with connect() as conn,conn.cursor() as cur:
|
||||
cur.execute("UPDATE minerops_agent_command_queue SET claimed_at=now() WHERE id=%s",(cid,))
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
self.assertEqual(sum(pool.map(lambda _: self.repo.retire_unsupported(AGENT,actor_user_id=ACTOR), range(2))),12)
|
||||
for n in range(12):
|
||||
audit=self.audits("stage-a-final-evidence-"+str(n)); self.assertEqual(len(audit),1)
|
||||
self.assertEqual(audit[0]["new_value_json"]["execution_outcome"],"unknown" if n%3 else "not_dispatched")
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Audited schedule retirement on an explicitly disposable PostgreSQL database."""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from app import main
|
||||
from app.agent_commands.exceptions import AgentCommandConflict
|
||||
from app.agent_commands.repository import AgentCommandRepository
|
||||
from app.agent_commands.policy import PHYSICAL_CONTROL_DISABLED
|
||||
from app.config import settings
|
||||
from app.identity import get_current_user, required_api_permission
|
||||
from tests.test_stage_a_containment_postgres import connect
|
||||
|
||||
|
||||
AGENT = "stage-a-schedule-agent"
|
||||
OTHER_AGENT = "stage-a-schedule-other-agent"
|
||||
ACTOR = UUID("00000000-0000-4000-8000-000000003301")
|
||||
SCHEDULE = "stage-a-schedule-history"
|
||||
|
||||
|
||||
class StageAScheduleCancellationTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if os.environ.get("MINEROPS_STAGE_A_DISPOSABLE") != "1" or not settings.postgres_db.startswith("minerops_stage_a_"):
|
||||
raise RuntimeError("Refusing schedule tests outside an explicitly disposable Stage A database")
|
||||
|
||||
def setUp(self):
|
||||
self.no_transport = patch("socket.create_connection", side_effect=AssertionError("No miner connection"))
|
||||
self.transport = self.no_transport.start()
|
||||
self.addCleanup(self.no_transport.stop)
|
||||
self.addCleanup(self.transport.assert_not_called)
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM audit_logs WHERE actor_user_id = %s OR old_value_json ->> 'agent_id' IN (%s, %s)",
|
||||
(ACTOR, AGENT, OTHER_AGENT))
|
||||
cur.execute("DELETE FROM minerops_agent_command_queue WHERE agent_id IN (%s, %s)", (AGENT, OTHER_AGENT))
|
||||
cur.execute("DELETE FROM minerops_scheduled_bulk_actions WHERE id = %s", (SCHEDULE,))
|
||||
cur.execute("""INSERT INTO users (id, username, display_name, password_hash, role_id)
|
||||
SELECT %s, 'stage-a-retirement-auditor', 'Retirement Auditor', 'unused', id
|
||||
FROM roles WHERE name = 'Admin' ON CONFLICT (id) DO NOTHING""", (ACTOR,))
|
||||
cur.execute("""INSERT INTO minerops_scheduled_bulk_actions
|
||||
(id, agent_id, name, target_type, scheduled_for, status, last_error)
|
||||
VALUES (%s, %s, 'Historical schedule', 'all', now(), 'running', 'prior evidence')""",
|
||||
(SCHEDULE, AGENT))
|
||||
self.repository = AgentCommandRepository(connect)
|
||||
|
||||
def seed(self, command_id, status="pending", agent=AGENT, kind="miner_control", schedule=SCHEDULE):
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO minerops_agent_command_queue
|
||||
(id, agent_id, command_type, command_status, request_payload, result_payload,
|
||||
error_message, claimed_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, 'prior evidence',
|
||||
CASE WHEN %s = 'claimed' THEN now() ELSE NULL END)""",
|
||||
(command_id, agent, kind, status,
|
||||
Jsonb({"scheduled_action_id": schedule, "ip_address": "192.0.2.1", "action": "suspend"}),
|
||||
Jsonb({"historical_result": True}), status))
|
||||
|
||||
@staticmethod
|
||||
def row(command_id):
|
||||
with connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT * FROM minerops_agent_command_queue WHERE id = %s", (command_id,))
|
||||
return dict(cur.fetchone())
|
||||
|
||||
@staticmethod
|
||||
def schedule():
|
||||
with connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT * FROM minerops_scheduled_bulk_actions WHERE id = %s", (SCHEDULE,))
|
||||
return dict(cur.fetchone())
|
||||
|
||||
@staticmethod
|
||||
def audits():
|
||||
with connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("""SELECT * FROM audit_logs WHERE actor_user_id = %s
|
||||
OR old_value_json ->> 'agent_id' IN (%s, %s)""", (ACTOR, AGENT, OTHER_AGENT))
|
||||
return [dict(row) for row in cur.fetchall()]
|
||||
|
||||
def retire(self, operation="cancel"):
|
||||
with patch.object(main, "get_db_connection", side_effect=connect):
|
||||
return getattr(main, f"{operation}_dashboard_scheduled_bulk_action")(
|
||||
SCHEDULE, {"id": str(ACTOR), "username": "retirement-auditor"},
|
||||
)
|
||||
|
||||
def assert_retired(self, command_id, before, operation):
|
||||
after = self.row(command_id)
|
||||
self.assertEqual(after["command_status"], "cancelled")
|
||||
for field in ("id", "agent_id", "command_type", "request_payload", "result_payload", "created_at", "claimed_at"):
|
||||
self.assertEqual(after[field], before[field], field)
|
||||
self.assertEqual(after["error_message"], "prior evidence\n" + PHYSICAL_CONTROL_DISABLED)
|
||||
self.assertIsNotNone(after["completed_at"])
|
||||
audits = [r for r in self.audits() if r["action"] == "agent.command.rejected"
|
||||
and r["old_value_json"]["command_id"] == command_id]
|
||||
self.assertEqual(len(audits), 1)
|
||||
audit = audits[0]
|
||||
self.assertEqual(audit["actor_user_id"], ACTOR)
|
||||
self.assertEqual(audit["old_value_json"]["agent_id"], AGENT)
|
||||
self.assertEqual(audit["old_value_json"]["status"], before["command_status"])
|
||||
self.assertEqual(audit["new_value_json"]["source"], "schedule_" + operation)
|
||||
self.assertEqual(audit["new_value_json"]["reason"], PHYSICAL_CONTROL_DISABLED)
|
||||
self.assertEqual(audit["new_value_json"]["execution_outcome"],
|
||||
"unknown")
|
||||
self.assertNotIn("192.0.2.1", str(audit))
|
||||
|
||||
def test_schedule_cancel_retires_pending_and_claimed_with_attributed_audits(self):
|
||||
for status in ("pending", "claimed"):
|
||||
command_id = "stage-a-schedule-cancel-" + status
|
||||
self.seed(command_id, status)
|
||||
before = {s: self.row("stage-a-schedule-cancel-" + s) for s in ("pending", "claimed")}
|
||||
self.retire()
|
||||
for status in before:
|
||||
self.assert_retired("stage-a-schedule-cancel-" + status, before[status], "cancel")
|
||||
self.assertEqual(self.schedule()["status"], "cancelled")
|
||||
self.assertEqual(len(self.audits()), 3)
|
||||
|
||||
def test_schedule_delete_is_audited_retirement_and_preserves_history(self):
|
||||
self.seed("stage-a-schedule-delete", "claimed")
|
||||
before, schedule_before = self.row("stage-a-schedule-delete"), self.schedule()
|
||||
self.retire("delete")
|
||||
self.assert_retired("stage-a-schedule-delete", before, "delete")
|
||||
after = self.schedule()
|
||||
for field in schedule_before:
|
||||
if field not in {"status", "last_error", "updated_at"}:
|
||||
self.assertEqual(after[field], schedule_before[field], field)
|
||||
parent_audits = [r for r in self.audits() if r["action"] == "scheduled_action.retired"]
|
||||
self.assertEqual(len(parent_audits), 1)
|
||||
self.assertEqual(parent_audits[0]["new_value_json"]["operation"], "delete")
|
||||
|
||||
def test_repeated_cancel_and_delete_are_idempotent(self):
|
||||
self.seed("stage-a-schedule-repeat")
|
||||
self.retire()
|
||||
before, schedule_before, audits_before = self.row("stage-a-schedule-repeat"), self.schedule(), self.audits()
|
||||
for operation in ("cancel", "delete", "delete"):
|
||||
self.retire(operation)
|
||||
self.assertEqual(self.row("stage-a-schedule-repeat"), before)
|
||||
self.assertEqual(self.schedule(), schedule_before)
|
||||
self.assertEqual(self.audits(), audits_before)
|
||||
|
||||
def test_terminal_queue_and_schedule_history_remains_immutable(self):
|
||||
for status in ("completed", "failed", "cancelled"):
|
||||
self.seed("stage-a-schedule-terminal-" + status, status)
|
||||
before = {s: self.row("stage-a-schedule-terminal-" + s) for s in ("completed", "failed", "cancelled")}
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("UPDATE minerops_scheduled_bulk_actions SET status = 'completed' WHERE id = %s", (SCHEDULE,))
|
||||
schedule_before = self.schedule()
|
||||
self.retire("delete")
|
||||
self.assertEqual(self.schedule(), schedule_before)
|
||||
for status, row in before.items():
|
||||
self.assertEqual(self.row("stage-a-schedule-terminal-" + status), row)
|
||||
self.assertEqual(self.audits(), [])
|
||||
|
||||
def test_stored_agent_and_schedule_scope_cannot_be_overridden(self):
|
||||
self.seed("stage-a-schedule-own")
|
||||
self.seed("stage-a-schedule-foreign", agent=OTHER_AGENT)
|
||||
self.seed("stage-a-schedule-unrelated", schedule="another-schedule")
|
||||
self.seed("stage-a-schedule-telemetry", kind="poll_known_miners_now")
|
||||
protected = {c: self.row(c) for c in ("stage-a-schedule-foreign", "stage-a-schedule-unrelated", "stage-a-schedule-telemetry")}
|
||||
self.retire()
|
||||
self.assertEqual(self.row("stage-a-schedule-own")["command_status"], "cancelled")
|
||||
for command_id, before in protected.items():
|
||||
self.assertEqual(self.row(command_id), before)
|
||||
|
||||
def test_schedule_unknown_type_is_rejected_with_its_actual_reason(self):
|
||||
self.seed("stage-a-schedule-unknown", kind="raw_api3")
|
||||
self.retire()
|
||||
rejection = next(a for a in self.audits() if a["action"] == "agent.command.rejected")
|
||||
self.assertEqual(rejection["new_value_json"]["reason"], "unsupported_command_schema_stage_a")
|
||||
|
||||
def test_concurrent_cancel_delete_workers_audit_each_transition_once(self):
|
||||
self.seed("stage-a-schedule-concurrent", "claimed")
|
||||
def worker(operation):
|
||||
return AgentCommandRepository(connect).retire_schedule(SCHEDULE, actor_user_id=str(ACTOR), operation=operation)
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
counts = list(pool.map(worker, ("cancel", "delete")))
|
||||
self.assertEqual(sorted(counts), [0, 1])
|
||||
self.assertEqual(self.row("stage-a-schedule-concurrent")["command_status"], "cancelled")
|
||||
self.assertEqual(len(self.audits()), 2)
|
||||
|
||||
def test_poll_finish_and_schedule_cancel_race_has_one_queue_audit(self):
|
||||
self.seed("stage-a-schedule-race", "claimed")
|
||||
def worker(i):
|
||||
repo = AgentCommandRepository(connect)
|
||||
try:
|
||||
if i == 0:
|
||||
return repo.retire_schedule(SCHEDULE, actor_user_id=str(ACTOR), operation="cancel")
|
||||
if i == 1:
|
||||
return repo.claim_pending(AGENT)
|
||||
return repo.finish_command("stage-a-schedule-race", AGENT, "completed", {}, None)
|
||||
except AgentCommandConflict:
|
||||
return None
|
||||
with ThreadPoolExecutor(max_workers=3) as pool:
|
||||
list(pool.map(worker, range(3)))
|
||||
rejections = [a for a in self.audits() if a["action"] == "agent.command.rejected"]
|
||||
self.assertEqual(len(rejections), 1)
|
||||
self.assertEqual(rejections[0]["new_value_json"]["execution_outcome"], "unknown")
|
||||
self.assertEqual(self.row("stage-a-schedule-race")["command_status"], "cancelled")
|
||||
|
||||
def test_audit_failure_after_one_insert_rolls_back_all_cancellations(self):
|
||||
for command_id in ("stage-a-schedule-rollback-a", "stage-a-schedule-rollback-b"):
|
||||
self.seed(command_id, "claimed")
|
||||
before = {c: self.row(c) for c in ("stage-a-schedule-rollback-a", "stage-a-schedule-rollback-b")}
|
||||
schedule_before = self.schedule()
|
||||
original = AgentCommandRepository._audit_rejection
|
||||
calls = []
|
||||
def fail_second(cur, row, **kwargs):
|
||||
calls.append(row["id"])
|
||||
if len(calls) == 2:
|
||||
raise RuntimeError("simulated audit failure")
|
||||
original(cur, row, **kwargs)
|
||||
with patch.object(AgentCommandRepository, "_audit_rejection", side_effect=fail_second):
|
||||
with self.assertRaisesRegex(RuntimeError, "simulated audit failure"):
|
||||
self.retire("delete")
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(self.schedule(), schedule_before)
|
||||
for command_id, row in before.items():
|
||||
self.assertEqual(self.row(command_id), row)
|
||||
self.assertEqual(self.audits(), [])
|
||||
|
||||
def test_schedule_parent_audit_failure_rolls_back_queue_and_parent(self):
|
||||
self.seed("stage-a-schedule-parent-rollback")
|
||||
before, parent_before = self.row("stage-a-schedule-parent-rollback"), self.schedule()
|
||||
# A database constraint failure after child audits exercises the final
|
||||
# schedule/audit portion, rather than only an early application exception.
|
||||
def connection_with_failure():
|
||||
conn = connect()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""CREATE OR REPLACE FUNCTION pg_temp.reject_schedule_audit() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$ BEGIN
|
||||
IF NEW.action = 'scheduled_action.retired' THEN RAISE EXCEPTION 'simulated parent audit failure'; END IF;
|
||||
RETURN NEW; END $$""")
|
||||
cur.execute("CREATE TRIGGER stage_a_parent_audit_failure BEFORE INSERT ON audit_logs FOR EACH ROW EXECUTE FUNCTION pg_temp.reject_schedule_audit()")
|
||||
# Trigger creation is in the very transaction the repository must roll back.
|
||||
return conn
|
||||
with self.assertRaisesRegex(Exception, "simulated parent audit failure"):
|
||||
AgentCommandRepository(connection_with_failure).retire_schedule(
|
||||
SCHEDULE, actor_user_id=str(ACTOR), operation="cancel")
|
||||
self.assertEqual(self.row("stage-a-schedule-parent-rollback"), before)
|
||||
self.assertEqual(self.schedule(), parent_before)
|
||||
self.assertEqual(self.audits(), [])
|
||||
|
||||
def test_unknown_schedule_has_no_queue_or_audit_side_effects(self):
|
||||
self.seed("stage-a-schedule-orphan")
|
||||
before = self.row("stage-a-schedule-orphan")
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM minerops_scheduled_bulk_actions WHERE id = %s", (SCHEDULE,))
|
||||
from fastapi import HTTPException
|
||||
with self.assertRaises(HTTPException) as error:
|
||||
self.retire()
|
||||
self.assertEqual(error.exception.status_code, 404)
|
||||
self.assertEqual(self.row("stage-a-schedule-orphan"), before)
|
||||
self.assertEqual(self.audits(), [])
|
||||
|
||||
def test_http_cancel_delete_require_control_permission(self):
|
||||
app = FastAPI()
|
||||
for operation in ("cancel", "delete"):
|
||||
path = f"/api/dashboard/scheduled-bulk-actions/{{action_id}}/{operation}"
|
||||
app.add_api_route(path, getattr(main, f"{operation}_dashboard_scheduled_bulk_action"), methods=["POST"])
|
||||
self.assertEqual(required_api_permission("POST", path), "miners.control")
|
||||
app.dependency_overrides[get_current_user] = lambda: {"id": str(ACTOR), "role": "Custom", "permissions": ["agents.manage"]}
|
||||
with TestClient(app) as client, patch.object(main, "get_db_connection") as db:
|
||||
for operation in ("cancel", "delete"):
|
||||
self.assertEqual(client.post(f"/api/dashboard/scheduled-bulk-actions/{SCHEDULE}/{operation}").status_code, 403)
|
||||
db.assert_not_called()
|
||||
|
||||
def test_offline_cutover_batch_is_audited_and_never_claims_readers(self):
|
||||
with connect() as conn, conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO minerops_agent_command_queue
|
||||
(id, agent_id, command_type, command_status, request_payload, claimed_at)
|
||||
SELECT 'stage-a-schedule-backlog-' || n, %s,
|
||||
CASE WHEN n %% 2 = 0 THEN 'miner_control' ELSE 'raw_api3' END,
|
||||
CASE WHEN n %% 3 = 0 THEN 'claimed' ELSE 'pending' END,
|
||||
'{}'::jsonb, CASE WHEN n %% 5 = 0 THEN now() ELSE NULL END
|
||||
FROM generate_series(1,1003) n""", (AGENT,))
|
||||
self.seed("stage-a-schedule-offline-read", kind="poll_known_miners_now")
|
||||
self.assertEqual(self.repository.retire_unsupported(AGENT, actor_user_id=str(ACTOR)), 1000)
|
||||
self.assertEqual(self.repository.retire_unsupported(AGENT, actor_user_id=str(ACTOR)), 3)
|
||||
self.assertEqual(self.repository.retire_unsupported(AGENT, actor_user_id=str(ACTOR)), 0)
|
||||
self.assertEqual(self.row("stage-a-schedule-offline-read")["command_status"], "pending")
|
||||
rejections = self.audits()
|
||||
self.assertEqual(len(rejections), 1003)
|
||||
for audit in rejections:
|
||||
self.assertEqual(audit["actor_user_id"], ACTOR)
|
||||
self.assertEqual(audit["new_value_json"]["source"], "stage_a_cutover")
|
||||
n = int(audit["old_value_json"]["command_id"].rsplit("-", 1)[1])
|
||||
self.assertEqual(audit["new_value_json"]["execution_outcome"],
|
||||
"unknown" if n % 3 == 0 or n % 5 == 0 else "not_dispatched")
|
||||
|
||||
def test_cutover_zero_batch_does_not_claim_a_locked_drain_is_complete(self):
|
||||
self.seed("stage-a-schedule-cutover-locked", "claimed")
|
||||
locker = connect()
|
||||
try:
|
||||
with locker.cursor() as cur:
|
||||
cur.execute("SELECT id FROM minerops_agent_command_queue WHERE id = %s FOR UPDATE", ("stage-a-schedule-cutover-locked",))
|
||||
self.assertEqual(self.repository.retire_unsupported(AGENT, actor_user_id=str(ACTOR)), 0)
|
||||
self.assertEqual(self.row("stage-a-schedule-cutover-locked")["command_status"], "claimed")
|
||||
finally:
|
||||
locker.rollback()
|
||||
locker.close()
|
||||
self.assertEqual(self.repository.retire_unsupported(AGENT, actor_user_id=str(ACTOR)), 1)
|
||||
self.assertEqual(self.row("stage-a-schedule-cutover-locked")["command_status"], "cancelled")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user