From d1a703da7cb7860e7546c455de135f80a616b1f1 Mon Sep 17 00:00:00 2001
From: MinerOps Admin <minerops@lussier.local>
Date: Wed, 22 Jul 2026 21:09:00 +0000
Subject: [PATCH] Add MinerOps operations runbook

---
 OPERATIONS.md | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 OPERATIONS.md

diff --git a/OPERATIONS.md b/OPERATIONS.md
new file mode 100644
index 0000000..57c7b54
--- /dev/null
+++ b/OPERATIONS.md
@@ -0,0 +1,115 @@
+# MinerOps Operations Runbook
+
+This file contains common day-to-day commands and safe procedures for operating MinerOps.
+
+## Quick health check
+
+Run from the Proxmox host:
+
+pct exec 306 -- bash -lc "curl -s http://127.0.0.1:8000/api/health"
+
+Expected result should include:
+
+status: ok
+database: ok
+redis: ok
+
+## Check Git status before changes
+
+pct exec 306 -- bash -lc "cd /opt/minerops && git status --short"
+
+If there is output, review it before changing files.
+
+## Commit and push a stable change
+
+pct exec 306 -- bash -lc "cd /opt/minerops && git add README.md CHANGELOG.md OPERATIONS.md backend scripts sql docker-compose.yml .gitignore && git commit -m 'Update MinerOps' && git push"
+
+## Rebuild backend
+
+Use this after changing backend/app/main.py or backend files.
+
+pct exec 306 -- bash -lc "cd /opt/minerops && docker compose build backend || docker-compose build backend"
+pct exec 306 -- bash -lc "cd /opt/minerops && docker compose up -d --force-recreate backend || docker-compose up -d --force-recreate backend"
+
+Then check:
+
+pct exec 306 -- bash -lc "docker exec minerops-backend python -m py_compile /app/app/main.py"
+pct exec 306 -- bash -lc "curl -s http://127.0.0.1:8000/api/health"
+
+## Check backend logs
+
+pct exec 306 -- bash -lc "docker logs minerops-backend --since 30m"
+
+For only errors:
+
+pct exec 306 -- bash -lc "docker logs minerops-backend --since 30m 2>&1 | grep -iE 'error|exception|traceback|500|programmingerror|keyerror|typeerror|warning' | tail -n 120 || true"
+
+## Main web pages
+
+Dashboard:
+https://plexus.lussier.cc/dashboard
+
+Repair Queue:
+https://plexus.lussier.cc/repairs
+
+Locations:
+https://plexus.lussier.cc/locations
+
+System Health:
+https://plexus.lussier.cc/system-health
+
+Gitea repo:
+https://gitea.lussier.cc/lussierj/minerops
+
+## Repair workflow
+
+1. Open /repairs.
+2. Search by MAC, IP, model, serial, or error.
+3. Add a repair note.
+4. Select part used and quantity if applicable.
+5. Set repair status.
+6. Use History to review, edit, or delete repair entries.
+7. Mark Ready to Shelf when repaired.
+8. Use Move to Shelf.
+9. Pick an open shelf position on /locations.
+
+## Retention check
+
+Open /system-health and review the Retention section.
+
+Also run:
+
+pct exec 306 -- bash -lc "tail -n 120 /opt/minerops/logs/status-retention.log || true"
+
+Check recent retention runs:
+
+pct exec 306 -- bash -lc "docker exec -i minerops-postgres psql -U minerops -d minerops -c \"SELECT mode, raw_status_rows_before, raw_status_rows_eligible_delete, raw_status_rows_deleted, hourly_rollup_rows_after, daily_rollup_rows_after, started_at, finished_at FROM minerops_retention_runs ORDER BY started_at DESC LIMIT 5;\""
+
+Retention is currently dry-run only. Do not switch to delete/apply mode until the dry-run has been reviewed.
+
+## Roll back to last Git version
+
+Check current status first:
+
+pct exec 306 -- bash -lc "cd /opt/minerops && git status --short"
+
+To discard uncommitted file changes:
+
+pct exec 306 -- bash -lc "cd /opt/minerops && git restore ."
+
+To reset to the last committed version:
+
+pct exec 306 -- bash -lc "cd /opt/minerops && git reset --hard HEAD"
+
+Then rebuild backend if backend files were involved.
+
+## Important safety rules
+
+- Do not commit .env files.
+- Do not commit database dumps.
+- Do not commit backups.
+- Do not commit logs.
+- Do not commit imports.
+- Check Git status before and after every patch.
+- Commit and push after every stable working change.
+- Use /system-health after major changes.