security: contain physical miner commands in Stage A

Disable physical control through legacy HTTP/M30 producers, typed admission,
queue dispatch and upgraded Windows executors. Restrict diagnostic transports
to exact read-only operations, with ordinal PowerShell validation before sockets.

Retire unsupported pending/claimed work with atomic, idempotent security audits.
Preserve prior delivery evidence and report unknown outcomes after a prior claim,
including explicitly repaired pending rows. Include containment, cancellation,
concurrency and malformed-payload regressions and the old-Agent cutover runbook.

Validation: independent final application-security revalidation PASS. Two unchanged
baseline source-assertion failures are documented as exceptions. Production release
still requires native Windows validation and verified fencing/stop/drain/upgrade;
server cancellation alone cannot revoke a cached command held by an old Agent.
This commit is contained in:
lussierj committed 2026-10-15 20:39:45 +00:00
1 parent 2d89e80410
commit e898f283ab
28 files changed
+2377 -2379

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
"""Stage A containment: no current payload authorizes a physical write.
There is deliberately no environment-variable override. Re-enabling physical
control requires a separately reviewed identity-binding protocol.
"""
PHYSICAL_CONTROL_DISABLED = "physical_control_disabled_stage_a"
UNSUPPORTED_COMMAND = "unsupported_command_schema_stage_a"
READ_ONLY_COMMAND_TYPES = (
"poll_known_miners_now",
"discover_new_miners",
"enrich_match_discovery",
)
def rejection_reason(command_type: str) -> str | None:
if command_type == "miner_control":
return PHYSICAL_CONTROL_DISABLED
if command_type not in READ_ONLY_COMMAND_TYPES:
return UNSUPPORTED_COMMAND
return None
def rejection_execution_outcome(row: dict) -> str:
"""A current pending state alone says nothing about earlier delivery.
The repository supplies an explicit pre-transition evidence check while
holding the queue row lock. Missing/ambiguous checks and result submissions
are unknown. This describes queue delivery, never proof of physical outcome.
"""
if (row.get("previous_status") == "pending"
and row.get("no_delivery_evidence") is True
and row.get("delivery_reported", False) is False):
return "not_dispatched"
return "unknown"
+193 -1
View File
@@ -8,6 +8,7 @@ from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from app.agent_commands.exceptions import AgentCommandConflict, AgentCommandNotFound
from app.agent_commands.policy import READ_ONLY_COMMAND_TYPES, rejection_execution_outcome, rejection_reason
from app.config import settings
@@ -19,6 +20,7 @@ WITH candidates AS (
FROM minerops_agent_command_queue
WHERE agent_id = %s
AND command_status = 'pending'
AND command_type = ANY(%s)
ORDER BY created_at, id
FOR UPDATE SKIP LOCKED
LIMIT %s
@@ -32,6 +34,70 @@ RETURNING queue.id AS command_id, queue.command_type,
queue.request_payload, queue.created_at
"""
# Inspect existing durable history before changing status/completed_at/error.
# A linked non-creation audit is conservatively ambiguous, even if timestamps
# were lost. Legacy results need not agree on Agent ID to prevent a false claim
# of non-delivery. Missing tables/permissions fail the transaction closed.
NO_DELIVERY_EVIDENCE_SQL = """
q.created_at IS NOT NULL
AND q.claimed_at IS NULL AND q.completed_at IS NULL
AND q.result_payload IS NULL AND COALESCE(q.error_message, '') = ''
AND NOT EXISTS (
SELECT 1 FROM minerops_agent_command_results r
WHERE r.command_id = q.id
OR (r.agent_id = q.agent_id AND NULLIF(btrim(r.command_id), '') IS NULL)
)
AND NOT EXISTS (
SELECT 1 FROM audit_logs a
WHERE a.action <> 'miner.control.queued'
AND (a.old_value_json ->> 'command_id' = q.id
OR a.new_value_json ->> 'command_id' = q.id
OR a.old_value_json ->> 'operation_id' = q.id
OR a.new_value_json ->> 'operation_id' = q.id
OR (a.entity_type = 'agent_command' AND a.entity_id::text = q.id))
)
"""
RETIRE_COMMANDS_SQL = f"""
WITH candidates AS (
SELECT q.id, q.command_status, ({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
FROM minerops_agent_command_queue q
WHERE agent_id = %s AND command_status IN ('pending', 'claimed')
AND NOT (command_type = ANY(%s))
ORDER BY created_at, id
FOR UPDATE SKIP LOCKED
LIMIT 1000
)
UPDATE minerops_agent_command_queue AS queue
SET command_status = 'cancelled', completed_at = COALESCE(queue.completed_at, now()),
error_message = concat_ws(chr(10), NULLIF(queue.error_message, ''), CASE WHEN queue.command_type = 'miner_control'
THEN 'physical_control_disabled_stage_a'
ELSE 'unsupported_command_schema_stage_a' END)
FROM candidates WHERE queue.id = candidates.id
RETURNING queue.id, queue.agent_id, queue.command_type,
candidates.command_status AS previous_status, candidates.no_delivery_evidence, queue.error_message
"""
RETIRE_SCHEDULE_COMMANDS_SQL = f"""
WITH candidates AS (
SELECT q.id, q.command_status, ({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
FROM minerops_agent_command_queue q
WHERE agent_id = %s AND request_payload ->> 'scheduled_action_id' = %s
AND command_status IN ('pending', 'claimed')
AND NOT (command_type = ANY(%s))
ORDER BY id
FOR UPDATE
)
UPDATE minerops_agent_command_queue AS queue
SET command_status = 'cancelled', completed_at = COALESCE(queue.completed_at, now()),
error_message = concat_ws(chr(10), NULLIF(queue.error_message, ''), CASE WHEN queue.command_type = 'miner_control'
THEN 'physical_control_disabled_stage_a'
ELSE 'unsupported_command_schema_stage_a' END)
FROM candidates WHERE queue.id = candidates.id
RETURNING queue.id, queue.agent_id, queue.command_type,
candidates.command_status AS previous_status, candidates.no_delivery_evidence
"""
def default_connection_factory() -> psycopg.Connection:
return psycopg.connect(
@@ -54,7 +120,8 @@ class AgentCommandRepository:
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
cur.execute(CLAIM_COMMANDS_SQL, (agent_id, limit))
self._retire_unsupported(cur, agent_id)
cur.execute(CLAIM_COMMANDS_SQL, (agent_id, list(READ_ONLY_COMMAND_TYPES), limit))
rows = [dict(row) for row in cur.fetchall()]
connection.commit()
return rows
@@ -64,6 +131,105 @@ class AgentCommandRepository:
finally:
connection.close()
@staticmethod
def _audit_rejection(
cur, row: dict, *, actor_user_id: str | None = None,
source: str = "agent_poll",
) -> None:
# Queue IDs can be legacy text, not UUIDs. Preserve them in JSON rather
# than coercing them into audit_logs.entity_id. Never log raw payloads.
cur.execute(
"""INSERT INTO audit_logs
(actor_user_id, action, entity_type, old_value_json, new_value_json)
VALUES (%s, 'agent.command.rejected', 'agent_command', %s, %s)""",
(actor_user_id, Jsonb({"command_id": row["id"], "agent_id": row["agent_id"],
"status": row["previous_status"]}),
Jsonb({"status": "cancelled", "reason": rejection_reason(row["command_type"]),
"source": source,
"execution_outcome": rejection_execution_outcome(row)})),
)
@classmethod
def _retire_unsupported(
cls, cur, agent_id: str, *, source: str = "agent_poll",
actor_user_id: str | None = None,
) -> int:
cur.execute(RETIRE_COMMANDS_SQL, (agent_id, list(READ_ONLY_COMMAND_TYPES)))
rows = list(cur.fetchall())
for row in rows:
cls._audit_rejection(cur, row, source=source, actor_user_id=actor_user_id)
return len(rows)
def retire_unsupported(self, agent_id: str, *, actor_user_id: str) -> int:
"""Audited cutover batch, including offline Agents; never claim any work.
A zero batch can mean locked rows were skipped. Operators must separately
verify no unsupported live rows remain before declaring a drain complete.
"""
if not actor_user_id:
raise ValueError("Cutover retirement requires an accountable operator actor")
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
count = self._retire_unsupported(cur, agent_id, source="stage_a_cutover",
actor_user_id=actor_user_id)
connection.commit()
return count
except Exception:
connection.rollback()
raise
finally:
connection.close()
def retire_schedule(self, schedule_id: str, *, actor_user_id: str, operation: str) -> int:
"""Cancel live unsupported work and retain schedule history atomically.
The stored schedule determines Agent scope. Delete retires the schedule;
neither operation erases evidence or changes terminal command history.
"""
if not actor_user_id or operation not in {"cancel", "delete"}:
raise ValueError("An authenticated actor and cancel/delete operation are required")
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
# Serialize retries and cancel/delete workers for this schedule.
cur.execute(
"""SELECT id, agent_id, status FROM minerops_scheduled_bulk_actions
WHERE id = %s FOR UPDATE""", (schedule_id,),
)
schedule = cur.fetchone()
if not schedule:
raise AgentCommandNotFound("Scheduled action not found")
cur.execute(RETIRE_SCHEDULE_COMMANDS_SQL,
(schedule["agent_id"], schedule_id, list(READ_ONLY_COMMAND_TYPES)))
rows = list(cur.fetchall())
for row in rows:
self._audit_rejection(cur, row, actor_user_id=actor_user_id,
source=f"schedule_{operation}")
if schedule["status"] in {"pending", "queued", "running"}:
cur.execute(
"""UPDATE minerops_scheduled_bulk_actions
SET status = 'cancelled', updated_at = now(),
last_error = concat_ws(chr(10), NULLIF(last_error, ''),
'physical_control_disabled_stage_a')
WHERE id = %s""", (schedule_id,),
)
cur.execute(
"""INSERT INTO audit_logs
(actor_user_id, action, entity_type, old_value_json, new_value_json)
VALUES (%s, 'scheduled_action.retired', 'scheduled_action', %s, %s)""",
(actor_user_id, Jsonb(dict(schedule)),
Jsonb({"status": "cancelled", "operation": operation,
"reason": "physical_control_disabled_stage_a"})),
)
connection.commit()
return len(rows)
except Exception:
connection.rollback()
raise
finally:
connection.close()
def get_command(self, command_id: str) -> dict | None:
connection = self._connection_factory()
try:
@@ -91,6 +257,32 @@ class AgentCommandRepository:
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
# Enforce the same boundary on results: an old claimed entry
# cannot be promoted by an old Agent, retry, or direct caller.
cur.execute(
f"""SELECT q.id, q.agent_id, q.command_type, q.command_status AS previous_status,
({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
FROM minerops_agent_command_queue q
WHERE q.id = %s AND q.agent_id = %s FOR UPDATE""",
(command_id, agent_id),
)
existing = cur.fetchone()
if existing and rejection_reason(existing["command_type"]):
reason = rejection_reason(existing["command_type"])
if existing["previous_status"] in {"pending", "claimed"}:
cur.execute(
"""UPDATE minerops_agent_command_queue
SET command_status = 'cancelled',
error_message = concat_ws(chr(10), NULLIF(error_message, ''), %s::text),
completed_at = COALESCE(completed_at, now()) WHERE id = %s""",
(reason, command_id),
)
# A result submission itself reports possible delivery, even
# when earlier durable claim metadata is absent.
existing["delivery_reported"] = True
self._audit_rejection(cur, existing, source="agent_result")
connection.commit()
raise AgentCommandConflict(reason)
cur.execute(
"""
UPDATE minerops_agent_command_queue
+1 -12
View File
@@ -1,6 +1,4 @@
"""HTTP boundary for authenticated Agent command transport."""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request
@@ -18,8 +16,6 @@ from app.agent_commands.models import (
)
from app.agent_commands.service import AgentCommandService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/agent/commands", tags=["agent-commands"])
AgentIdentity = Annotated[str, Depends(require_agent_identity)]
AgentService = Annotated[AgentCommandService, Depends(get_agent_command_service)]
@@ -29,14 +25,7 @@ AgentService = Annotated[AgentCommandService, Depends(get_agent_command_service)
def claim_pending_commands(
request: Request, agent_id: AgentIdentity, service: AgentService
) -> AgentCommandListResponse:
before_claim = getattr(
request.app.state, "agent_command_before_claim", None
)
if callable(before_claim):
try:
before_claim()
except Exception:
logger.exception("Scheduled command materialization failed before Agent claim")
del request
return AgentCommandListResponse(commands=service.claim(agent_id))
+10 -32
View File
@@ -2,19 +2,16 @@
import re
from pydantic import ValidationError
from app.agent_commands.exceptions import (
AgentCommandConflict,
AgentCommandNotFound,
AgentResultValidationError,
)
from app.agent_commands.models import (
AgentCommand,
AgentCommandResultRequest,
MinerControlResultPayload,
)
from app.agent_commands.repository import AgentCommandRepository
from app.agent_commands.policy import rejection_reason
class AgentCommandService:
@@ -25,6 +22,7 @@ class AgentCommandService:
return [
AgentCommand.model_validate(row)
for row in self.repository.claim_pending(agent_id)
if rejection_reason(row["command_type"]) is None
]
@staticmethod
@@ -42,6 +40,14 @@ class AgentCommandService:
raise AgentCommandNotFound("Agent command not found")
if command["agent_id"] != agent_id:
raise AgentCommandConflict("Agent command belongs to another Agent")
reason = rejection_reason(command["command_type"])
if reason:
# Repository repeats this check under a row lock and records the
# cancellation atomically; never accept an old physical result.
self.repository.finish_command(
request.command_id, agent_id, "cancelled", {}, reason
)
raise AgentCommandConflict(reason)
if command["command_status"] != "claimed":
raise AgentCommandConflict(
f"Agent command is {command['command_status']}, not claimed"
@@ -49,34 +55,6 @@ class AgentCommandService:
result_payload = request.result_payload
error_message = self._legacy_error(request.error_message)
if command["command_type"] == "miner_control":
try:
result = MinerControlResultPayload.model_validate(request.result_payload)
except ValidationError as exc:
raise AgentResultValidationError(
"Invalid typed miner-control result"
) from exc
if request.status == "completed" and not result.command_accepted:
raise AgentResultValidationError(
"Completed miner-control results must be accepted"
)
if request.status == "failed" and result.command_accepted:
raise AgentResultValidationError(
"Failed miner-control results cannot be accepted"
)
action = (command.get("request_payload") or {}).get("action")
if (
action in {"suspend", "resume"}
and result.command_accepted
and result.verification_status != "unconfirmed"
):
raise AgentResultValidationError(
"Accepted suspend and resume results must remain unconfirmed"
)
result_payload = result.model_dump(mode="json")
error_message = (
result.failure_code.replace("_", " ") if result.failure_code else None
)
self.repository.finish_command(
request.command_id,
+15
View File
@@ -708,6 +708,8 @@ def machine_agent_request(path: str) -> bool:
"/api/agent/pool-snapshots",
"/api/agent/commands/pending",
"/api/agent/commands/result",
"/api/agent/commands/pending-disabled-old",
"/api/agent/commands/result-disabled-old",
"/api/agent/discoveries",
"/api/agent/miner-enrichment",
"/api/agent/miner-enrichment-v2",
@@ -760,6 +762,19 @@ def component_request(path: str) -> bool:
def required_api_permission(method: str, path: str) -> str | None:
parts = path.split("/")
if method == "POST" and path == "/api/dashboard/m30-command-test-lab/run":
return "miners.control"
if method == "POST" and path == "/api/tools/whatsminer-read-test":
return "miners.control"
if method == "POST" and path in {
"/api/dashboard/scheduled-bulk-actions",
"/api/dashboard/scheduled-bulk-actions/process-due",
}:
return "miners.control"
if (method == "POST" and len(parts) == 6
and parts[:4] == ["", "api", "dashboard", "scheduled-bulk-actions"]
and parts[5] in {"cancel", "delete"}):
return "miners.control"
if (
len(parts) >= 5
and parts[:3] == ["", "api", "miners"]
+147 -1589
View File
File diff suppressed because it is too large. Load diff
+5 -76
View File
@@ -1,7 +1,6 @@
"""Business rules for typed, Agent-routed miner controls."""
from typing import Any
from uuid import UUID, uuid4
from uuid import UUID
from app.miner_control.exceptions import (
ControlledMinerNotFound,
@@ -16,6 +15,7 @@ from app.miner_control.models import (
MinerControlProjection,
)
from app.miner_control.repository import MinerControlRepository
from app.agent_commands.policy import PHYSICAL_CONTROL_DISABLED
KNOWN_MODES = {"low", "normal", "high"}
@@ -29,33 +29,8 @@ class MinerControlService:
@staticmethod
def _capability(context: dict) -> ControlCapability:
if not context.get("agent_zone_id"):
return ControlCapability(state="unavailable", reason="agent_zone_unassigned")
if context.get("agent_zone_exists") is not True:
return ControlCapability(state="unavailable", reason="agent_zone_missing")
if context.get("agent_zone_enabled") is not True:
return ControlCapability(state="unavailable", reason="agent_zone_disabled")
if not context.get("agent_id"):
return ControlCapability(state="unavailable", reason="agent_unavailable")
if not context.get("current_ip"):
return ControlCapability(state="unavailable", reason="current_ip_missing")
if context.get("api3_supported") is False:
return ControlCapability(state="unavailable", reason="api3_unsupported")
if context.get("api3_supported") is not True:
return ControlCapability(state="unknown", reason="api3_support_unknown")
payload = context.get("last_payload") or {}
if payload.get("control_available") is False:
reason = payload.get("control_unavailable_reason")
return ControlCapability(
state="unavailable",
reason=reason if reason == "credentials_missing" else "agent_unavailable",
)
if payload.get("control_available") is not True:
return ControlCapability(
state="unknown", reason="agent_capability_unknown"
)
return ControlCapability(state="supported")
# Credentials and Zone routing are not physical identity authorization.
return ControlCapability(state="unavailable", reason=PHYSICAL_CONTROL_DISABLED)
@staticmethod
def _operation(miner_id: UUID, row: dict) -> MinerControlOperation:
@@ -120,50 +95,4 @@ class MinerControlService:
) -> MinerControlAccepted:
if not actor_user_id:
raise MinerControlActorRequired("Authenticated actor UUID is required")
context = self.repository.get_control_context(miner_id)
if not context:
raise ControlledMinerNotFound("Miner not found")
capability = self._capability(context)
if capability.state != "supported":
raise MinerControlUnavailable(
capability.reason or "control_capability_not_supported"
)
runtime = str(context.get("runtime_state") or "").strip().lower()
if action == "suspend" and runtime not in RUNNING_STATES:
raise MinerControlUnavailable("miner_not_factually_running")
if action == "resume" and runtime not in STOPPED_STATES:
raise MinerControlUnavailable("miner_not_factually_stopped")
if action == "performance_mode" and requested_value not in KNOWN_MODES:
raise MinerControlUnavailable("invalid_performance_mode")
operation_id = str(uuid4())
payload: dict[str, Any] = {
"schema_version": 1,
"miner_id": str(miner_id),
"action": action,
"requested_value": requested_value,
"ip_address": context["current_ip"],
}
operation = self.repository.enqueue_with_audit(
operation_id=operation_id,
miner_id=miner_id,
agent_id=context["agent_id"],
actor_user_id=actor_user_id,
actor_username=actor_username,
request_payload=payload,
audit_payload={
"operation_id": operation_id,
"miner_id": str(miner_id),
"action": action,
"requested_value": requested_value,
},
)
return MinerControlAccepted(
operation_id=operation["operation_id"],
miner_id=miner_id,
action=action,
requested_value=requested_value,
status=operation["status"],
created_at=operation["created_at"],
)
raise MinerControlUnavailable(PHYSICAL_CONTROL_DISABLED)