security: contain physical miner commands in Stage A

Disable physical control through legacy HTTP/M30 producers, typed admission,
queue dispatch and upgraded Windows executors. Restrict diagnostic transports
to exact read-only operations, with ordinal PowerShell validation before sockets.

Retire unsupported pending/claimed work with atomic, idempotent security audits.
Preserve prior delivery evidence and report unknown outcomes after a prior claim,
including explicitly repaired pending rows. Include containment, cancellation,
concurrency and malformed-payload regressions and the old-Agent cutover runbook.

Validation: independent final application-security revalidation PASS. Two unchanged
baseline source-assertion failures are documented as exceptions. Production release
still requires native Windows validation and verified fencing/stop/drain/upgrade;
server cancellation alone cannot revoke a cached command held by an old Agent.
This commit is contained in:
lussierj committed 2026-10-15 20:39:45 +00:00
1 parent 2d89e80410
commit e898f283ab
28 files changed
+2377 -2379

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
"""Stage A containment: no current payload authorizes a physical write.
There is deliberately no environment-variable override. Re-enabling physical
control requires a separately reviewed identity-binding protocol.
"""
PHYSICAL_CONTROL_DISABLED = "physical_control_disabled_stage_a"
UNSUPPORTED_COMMAND = "unsupported_command_schema_stage_a"
READ_ONLY_COMMAND_TYPES = (
"poll_known_miners_now",
"discover_new_miners",
"enrich_match_discovery",
)
def rejection_reason(command_type: str) -> str | None:
if command_type == "miner_control":
return PHYSICAL_CONTROL_DISABLED
if command_type not in READ_ONLY_COMMAND_TYPES:
return UNSUPPORTED_COMMAND
return None
def rejection_execution_outcome(row: dict) -> str:
"""A current pending state alone says nothing about earlier delivery.
The repository supplies an explicit pre-transition evidence check while
holding the queue row lock. Missing/ambiguous checks and result submissions
are unknown. This describes queue delivery, never proof of physical outcome.
"""
if (row.get("previous_status") == "pending"
and row.get("no_delivery_evidence") is True
and row.get("delivery_reported", False) is False):
return "not_dispatched"
return "unknown"
+193 -1
View File
@@ -8,6 +8,7 @@ from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from app.agent_commands.exceptions import AgentCommandConflict, AgentCommandNotFound
from app.agent_commands.policy import READ_ONLY_COMMAND_TYPES, rejection_execution_outcome, rejection_reason
from app.config import settings
@@ -19,6 +20,7 @@ WITH candidates AS (
FROM minerops_agent_command_queue
WHERE agent_id = %s
AND command_status = 'pending'
AND command_type = ANY(%s)
ORDER BY created_at, id
FOR UPDATE SKIP LOCKED
LIMIT %s
@@ -32,6 +34,70 @@ RETURNING queue.id AS command_id, queue.command_type,
queue.request_payload, queue.created_at
"""
# Inspect existing durable history before changing status/completed_at/error.
# A linked non-creation audit is conservatively ambiguous, even if timestamps
# were lost. Legacy results need not agree on Agent ID to prevent a false claim
# of non-delivery. Missing tables/permissions fail the transaction closed.
NO_DELIVERY_EVIDENCE_SQL = """
q.created_at IS NOT NULL
AND q.claimed_at IS NULL AND q.completed_at IS NULL
AND q.result_payload IS NULL AND COALESCE(q.error_message, '') = ''
AND NOT EXISTS (
SELECT 1 FROM minerops_agent_command_results r
WHERE r.command_id = q.id
OR (r.agent_id = q.agent_id AND NULLIF(btrim(r.command_id), '') IS NULL)
)
AND NOT EXISTS (
SELECT 1 FROM audit_logs a
WHERE a.action <> 'miner.control.queued'
AND (a.old_value_json ->> 'command_id' = q.id
OR a.new_value_json ->> 'command_id' = q.id
OR a.old_value_json ->> 'operation_id' = q.id
OR a.new_value_json ->> 'operation_id' = q.id
OR (a.entity_type = 'agent_command' AND a.entity_id::text = q.id))
)
"""
RETIRE_COMMANDS_SQL = f"""
WITH candidates AS (
SELECT q.id, q.command_status, ({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
FROM minerops_agent_command_queue q
WHERE agent_id = %s AND command_status IN ('pending', 'claimed')
AND NOT (command_type = ANY(%s))
ORDER BY created_at, id
FOR UPDATE SKIP LOCKED
LIMIT 1000
)
UPDATE minerops_agent_command_queue AS queue
SET command_status = 'cancelled', completed_at = COALESCE(queue.completed_at, now()),
error_message = concat_ws(chr(10), NULLIF(queue.error_message, ''), CASE WHEN queue.command_type = 'miner_control'
THEN 'physical_control_disabled_stage_a'
ELSE 'unsupported_command_schema_stage_a' END)
FROM candidates WHERE queue.id = candidates.id
RETURNING queue.id, queue.agent_id, queue.command_type,
candidates.command_status AS previous_status, candidates.no_delivery_evidence, queue.error_message
"""
RETIRE_SCHEDULE_COMMANDS_SQL = f"""
WITH candidates AS (
SELECT q.id, q.command_status, ({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
FROM minerops_agent_command_queue q
WHERE agent_id = %s AND request_payload ->> 'scheduled_action_id' = %s
AND command_status IN ('pending', 'claimed')
AND NOT (command_type = ANY(%s))
ORDER BY id
FOR UPDATE
)
UPDATE minerops_agent_command_queue AS queue
SET command_status = 'cancelled', completed_at = COALESCE(queue.completed_at, now()),
error_message = concat_ws(chr(10), NULLIF(queue.error_message, ''), CASE WHEN queue.command_type = 'miner_control'
THEN 'physical_control_disabled_stage_a'
ELSE 'unsupported_command_schema_stage_a' END)
FROM candidates WHERE queue.id = candidates.id
RETURNING queue.id, queue.agent_id, queue.command_type,
candidates.command_status AS previous_status, candidates.no_delivery_evidence
"""
def default_connection_factory() -> psycopg.Connection:
return psycopg.connect(
@@ -54,7 +120,8 @@ class AgentCommandRepository:
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
cur.execute(CLAIM_COMMANDS_SQL, (agent_id, limit))
self._retire_unsupported(cur, agent_id)
cur.execute(CLAIM_COMMANDS_SQL, (agent_id, list(READ_ONLY_COMMAND_TYPES), limit))
rows = [dict(row) for row in cur.fetchall()]
connection.commit()
return rows
@@ -64,6 +131,105 @@ class AgentCommandRepository:
finally:
connection.close()
@staticmethod
def _audit_rejection(
cur, row: dict, *, actor_user_id: str | None = None,
source: str = "agent_poll",
) -> None:
# Queue IDs can be legacy text, not UUIDs. Preserve them in JSON rather
# than coercing them into audit_logs.entity_id. Never log raw payloads.
cur.execute(
"""INSERT INTO audit_logs
(actor_user_id, action, entity_type, old_value_json, new_value_json)
VALUES (%s, 'agent.command.rejected', 'agent_command', %s, %s)""",
(actor_user_id, Jsonb({"command_id": row["id"], "agent_id": row["agent_id"],
"status": row["previous_status"]}),
Jsonb({"status": "cancelled", "reason": rejection_reason(row["command_type"]),
"source": source,
"execution_outcome": rejection_execution_outcome(row)})),
)
@classmethod
def _retire_unsupported(
cls, cur, agent_id: str, *, source: str = "agent_poll",
actor_user_id: str | None = None,
) -> int:
cur.execute(RETIRE_COMMANDS_SQL, (agent_id, list(READ_ONLY_COMMAND_TYPES)))
rows = list(cur.fetchall())
for row in rows:
cls._audit_rejection(cur, row, source=source, actor_user_id=actor_user_id)
return len(rows)
def retire_unsupported(self, agent_id: str, *, actor_user_id: str) -> int:
"""Audited cutover batch, including offline Agents; never claim any work.
A zero batch can mean locked rows were skipped. Operators must separately
verify no unsupported live rows remain before declaring a drain complete.
"""
if not actor_user_id:
raise ValueError("Cutover retirement requires an accountable operator actor")
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
count = self._retire_unsupported(cur, agent_id, source="stage_a_cutover",
actor_user_id=actor_user_id)
connection.commit()
return count
except Exception:
connection.rollback()
raise
finally:
connection.close()
def retire_schedule(self, schedule_id: str, *, actor_user_id: str, operation: str) -> int:
"""Cancel live unsupported work and retain schedule history atomically.
The stored schedule determines Agent scope. Delete retires the schedule;
neither operation erases evidence or changes terminal command history.
"""
if not actor_user_id or operation not in {"cancel", "delete"}:
raise ValueError("An authenticated actor and cancel/delete operation are required")
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
# Serialize retries and cancel/delete workers for this schedule.
cur.execute(
"""SELECT id, agent_id, status FROM minerops_scheduled_bulk_actions
WHERE id = %s FOR UPDATE""", (schedule_id,),
)
schedule = cur.fetchone()
if not schedule:
raise AgentCommandNotFound("Scheduled action not found")
cur.execute(RETIRE_SCHEDULE_COMMANDS_SQL,
(schedule["agent_id"], schedule_id, list(READ_ONLY_COMMAND_TYPES)))
rows = list(cur.fetchall())
for row in rows:
self._audit_rejection(cur, row, actor_user_id=actor_user_id,
source=f"schedule_{operation}")
if schedule["status"] in {"pending", "queued", "running"}:
cur.execute(
"""UPDATE minerops_scheduled_bulk_actions
SET status = 'cancelled', updated_at = now(),
last_error = concat_ws(chr(10), NULLIF(last_error, ''),
'physical_control_disabled_stage_a')
WHERE id = %s""", (schedule_id,),
)
cur.execute(
"""INSERT INTO audit_logs
(actor_user_id, action, entity_type, old_value_json, new_value_json)
VALUES (%s, 'scheduled_action.retired', 'scheduled_action', %s, %s)""",
(actor_user_id, Jsonb(dict(schedule)),
Jsonb({"status": "cancelled", "operation": operation,
"reason": "physical_control_disabled_stage_a"})),
)
connection.commit()
return len(rows)
except Exception:
connection.rollback()
raise
finally:
connection.close()
def get_command(self, command_id: str) -> dict | None:
connection = self._connection_factory()
try:
@@ -91,6 +257,32 @@ class AgentCommandRepository:
connection = self._connection_factory()
try:
with connection.cursor(row_factory=dict_row) as cur:
# Enforce the same boundary on results: an old claimed entry
# cannot be promoted by an old Agent, retry, or direct caller.
cur.execute(
f"""SELECT q.id, q.agent_id, q.command_type, q.command_status AS previous_status,
({NO_DELIVERY_EVIDENCE_SQL}) AS no_delivery_evidence
FROM minerops_agent_command_queue q
WHERE q.id = %s AND q.agent_id = %s FOR UPDATE""",
(command_id, agent_id),
)
existing = cur.fetchone()
if existing and rejection_reason(existing["command_type"]):
reason = rejection_reason(existing["command_type"])
if existing["previous_status"] in {"pending", "claimed"}:
cur.execute(
"""UPDATE minerops_agent_command_queue
SET command_status = 'cancelled',
error_message = concat_ws(chr(10), NULLIF(error_message, ''), %s::text),
completed_at = COALESCE(completed_at, now()) WHERE id = %s""",
(reason, command_id),
)
# A result submission itself reports possible delivery, even
# when earlier durable claim metadata is absent.
existing["delivery_reported"] = True
self._audit_rejection(cur, existing, source="agent_result")
connection.commit()
raise AgentCommandConflict(reason)
cur.execute(
"""
UPDATE minerops_agent_command_queue
+1 -12
View File
@@ -1,6 +1,4 @@
"""HTTP boundary for authenticated Agent command transport."""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request
@@ -18,8 +16,6 @@ from app.agent_commands.models import (
)
from app.agent_commands.service import AgentCommandService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/agent/commands", tags=["agent-commands"])
AgentIdentity = Annotated[str, Depends(require_agent_identity)]
AgentService = Annotated[AgentCommandService, Depends(get_agent_command_service)]
@@ -29,14 +25,7 @@ AgentService = Annotated[AgentCommandService, Depends(get_agent_command_service)
def claim_pending_commands(
request: Request, agent_id: AgentIdentity, service: AgentService
) -> AgentCommandListResponse:
before_claim = getattr(
request.app.state, "agent_command_before_claim", None
)
if callable(before_claim):
try:
before_claim()
except Exception:
logger.exception("Scheduled command materialization failed before Agent claim")
del request
return AgentCommandListResponse(commands=service.claim(agent_id))
+10 -32
View File
@@ -2,19 +2,16 @@
import re
from pydantic import ValidationError
from app.agent_commands.exceptions import (
AgentCommandConflict,
AgentCommandNotFound,
AgentResultValidationError,
)
from app.agent_commands.models import (
AgentCommand,
AgentCommandResultRequest,
MinerControlResultPayload,
)
from app.agent_commands.repository import AgentCommandRepository
from app.agent_commands.policy import rejection_reason
class AgentCommandService:
@@ -25,6 +22,7 @@ class AgentCommandService:
return [
AgentCommand.model_validate(row)
for row in self.repository.claim_pending(agent_id)
if rejection_reason(row["command_type"]) is None
]
@staticmethod
@@ -42,6 +40,14 @@ class AgentCommandService:
raise AgentCommandNotFound("Agent command not found")
if command["agent_id"] != agent_id:
raise AgentCommandConflict("Agent command belongs to another Agent")
reason = rejection_reason(command["command_type"])
if reason:
# Repository repeats this check under a row lock and records the
# cancellation atomically; never accept an old physical result.
self.repository.finish_command(
request.command_id, agent_id, "cancelled", {}, reason
)
raise AgentCommandConflict(reason)
if command["command_status"] != "claimed":
raise AgentCommandConflict(
f"Agent command is {command['command_status']}, not claimed"
@@ -49,34 +55,6 @@ class AgentCommandService:
result_payload = request.result_payload
error_message = self._legacy_error(request.error_message)
if command["command_type"] == "miner_control":
try:
result = MinerControlResultPayload.model_validate(request.result_payload)
except ValidationError as exc:
raise AgentResultValidationError(
"Invalid typed miner-control result"
) from exc
if request.status == "completed" and not result.command_accepted:
raise AgentResultValidationError(
"Completed miner-control results must be accepted"
)
if request.status == "failed" and result.command_accepted:
raise AgentResultValidationError(
"Failed miner-control results cannot be accepted"
)
action = (command.get("request_payload") or {}).get("action")
if (
action in {"suspend", "resume"}
and result.command_accepted
and result.verification_status != "unconfirmed"
):
raise AgentResultValidationError(
"Accepted suspend and resume results must remain unconfirmed"
)
result_payload = result.model_dump(mode="json")
error_message = (
result.failure_code.replace("_", " ") if result.failure_code else None
)
self.repository.finish_command(
request.command_id,