From d4df20b7402a9dc32cf065a35f610b3c1d3979b1 Mon Sep 17 00:00:00 2001 From: JC Date: Tue, 28 Jul 2026 15:38:52 +0000 Subject: [PATCH] Add manual miner inventory schema --- .../versions/0008_manual_miner_inventory.py | 193 +++++++++ scripts/test-dev-baseline.sh | 2 +- scripts/test-manual-inventory-migration.sh | 398 ++++++++++++++++++ sql/verify-development-schema.sql | 2 + 4 files changed, 594 insertions(+), 1 deletion(-) create mode 100644 backend/alembic/versions/0008_manual_miner_inventory.py create mode 100755 scripts/test-manual-inventory-migration.sh diff --git a/backend/alembic/versions/0008_manual_miner_inventory.py b/backend/alembic/versions/0008_manual_miner_inventory.py new file mode 100644 index 0000000..9f1cafc --- /dev/null +++ b/backend/alembic/versions/0008_manual_miner_inventory.py @@ -0,0 +1,193 @@ +"""add manual miner inventory schema + +Revision ID: 0008_manual_miner_inventory +Revises: 0007_identity_and_roles +Create Date: 2026-07-28 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + + +revision: str = "0008_manual_miner_inventory" +down_revision: Union[str, None] = "0007_identity_and_roles" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +TAG_TRIMMED_NAME_SQL = ( + "regexp_replace(name, '^[[:space:]]+|[[:space:]]+$', '', 'g')" +) + + +def upgrade() -> None: + op.add_column("miners", sa.Column("control_board_sn", sa.Text(), nullable=True)) + op.add_column( + "miners", + sa.Column( + "lifecycle_status", + sa.Text(), + nullable=False, + server_default=sa.text("'active'"), + ), + ) + op.create_check_constraint( + "ck_miners_lifecycle_status", + "miners", + "lifecycle_status IN ('active', 'spare', 'retired')", + ) + + op.execute( + """ + DO $$ + DECLARE + conflicting_asset_ids TEXT; + BEGIN + SELECT string_agg( + format('%L (%s records)', normalized_asset_id, record_count), + ', ' ORDER BY normalized_asset_id + ) + INTO conflicting_asset_ids + FROM ( + SELECT + lower(trim(asset_tag)) AS normalized_asset_id, + COUNT(*) AS record_count + FROM miners + WHERE asset_tag IS NOT NULL + AND trim(asset_tag) <> '' + GROUP BY lower(trim(asset_tag)) + HAVING COUNT(*) > 1 + ) conflicts; + + IF conflicting_asset_ids IS NOT NULL THEN + RAISE EXCEPTION USING + MESSAGE = 'Cannot enforce globally unique Asset IDs. ' + || 'Conflicting normalized values: ' + || conflicting_asset_ids, + HINT = 'Correct the conflicting miners.asset_tag values ' + || 'before upgrading, then run the migration again.'; + END IF; + END + $$; + """ + ) + op.create_index( + "uq_miners_asset_tag_ci", + "miners", + [sa.text("lower(trim(asset_tag))")], + unique=True, + postgresql_where=sa.text("asset_tag IS NOT NULL AND trim(asset_tag) <> ''"), + ) + + op.create_table( + "tags", + sa.Column( + "id", + postgresql.UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column("name", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.CheckConstraint( + f"{TAG_TRIMMED_NAME_SQL} <> ''", + name="ck_tags_name_not_blank", + ), + ) + op.create_index( + "uq_tags_name_ci", + "tags", + [sa.text(f"lower({TAG_TRIMMED_NAME_SQL})")], + unique=True, + ) + + op.create_table( + "miner_tags", + sa.Column( + "miner_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("miners.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "tag_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("tags.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.PrimaryKeyConstraint("miner_id", "tag_id", name="pk_miner_tags"), + ) + op.create_index("idx_miner_tags_tag_id", "miner_tags", ["tag_id"]) + + # Discovery continues to receive now() defaults. Manual creation can + # explicitly store NULL to represent an asset that has never been observed. + op.alter_column( + "miners", + "first_seen_at", + existing_type=sa.DateTime(timezone=True), + existing_server_default=sa.text("now()"), + nullable=True, + ) + op.alter_column( + "miners", + "last_seen_at", + existing_type=sa.DateTime(timezone=True), + existing_server_default=sa.text("now()"), + nullable=True, + ) + + +def downgrade() -> None: + # created_at is non-null in the pre-0008 schema, making this deterministic. + op.execute( + """ + UPDATE miners + SET first_seen_at = COALESCE(first_seen_at, created_at), + last_seen_at = COALESCE(last_seen_at, created_at) + WHERE first_seen_at IS NULL + OR last_seen_at IS NULL; + """ + ) + op.alter_column( + "miners", + "last_seen_at", + existing_type=sa.DateTime(timezone=True), + existing_server_default=sa.text("now()"), + nullable=False, + ) + op.alter_column( + "miners", + "first_seen_at", + existing_type=sa.DateTime(timezone=True), + existing_server_default=sa.text("now()"), + nullable=False, + ) + + op.drop_index("idx_miner_tags_tag_id", table_name="miner_tags") + op.drop_table("miner_tags") + op.drop_index("uq_tags_name_ci", table_name="tags") + op.drop_table("tags") + + op.drop_index("uq_miners_asset_tag_ci", table_name="miners") + op.drop_constraint("ck_miners_lifecycle_status", "miners", type_="check") + op.drop_column("miners", "lifecycle_status") + op.drop_column("miners", "control_board_sn") diff --git a/scripts/test-dev-baseline.sh b/scripts/test-dev-baseline.sh index 3e35396..afaee47 100755 --- a/scripts/test-dev-baseline.sh +++ b/scripts/test-dev-baseline.sh @@ -88,7 +88,7 @@ health_json="$(cat "${health_file}")" echo "Checking migrations and required schema..." current_revision="$("${compose[@]}" exec -T backend alembic current 2>/dev/null)" -[[ "${current_revision}" == *"0007_identity_and_roles"* ]] +[[ "${current_revision}" == *"0008_manual_miner_inventory"* ]] missing_tables="$( "${compose[@]}" exec -T postgres sh -c \ diff --git a/scripts/test-manual-inventory-migration.sh b/scripts/test-manual-inventory-migration.sh new file mode 100755 index 0000000..34d63e8 --- /dev/null +++ b/scripts/test-manual-inventory-migration.sh @@ -0,0 +1,398 @@ +#!/usr/bin/env bash +set -euo pipefail + +project_name="${MINEROPS_INVENTORY_MIGRATION_PROJECT:-minerops-inventory-migration-$$}" +compose=( + docker compose + --project-name "${project_name}" + --env-file .env.dev + -f compose.dev.yaml +) +empty_database="minerops_inventory_empty" + +cleanup() { + status=$? + if (( status != 0 )); then + echo + echo "Migration test failed. Recent PostgreSQL logs:" + "${compose[@]}" logs --no-color --tail 120 postgres || true + fi + "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true + exit "${status}" +} + +trap cleanup EXIT + +if [[ ! -f .env.dev ]]; then + echo "Missing .env.dev. Run ./scripts/dev-env-init.sh first." >&2 + exit 1 +fi + +run_alembic() { + "${compose[@]}" run --rm --no-deps backend alembic "$@" +} + +run_empty_alembic() { + "${compose[@]}" run --rm --no-deps \ + -e "POSTGRES_DB=${empty_database}" \ + backend alembic "$@" +} + +echo "Starting disposable PostgreSQL..." +"${compose[@]}" up --detach postgres +for attempt in $(seq 1 60); do + if "${compose[@]}" exec -T postgres sh -ec \ + 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null 2>&1; then + break + fi + if (( attempt == 60 )); then + echo "PostgreSQL did not become ready." >&2 + exit 1 + fi + sleep 1 +done + +echo "Building migration image..." +"${compose[@]}" build backend + +echo "Upgrading to the pre-inventory schema..." +run_alembic upgrade 0007_identity_and_roles + +echo "Seeding existing miner data and conflicting Asset IDs..." +"${compose[@]}" exec -T postgres sh -ec \ + 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL' +INSERT INTO miners ( + id, + asset_tag, + current_mac, + current_ip, + miner_sn, + power_sn, + status, + first_seen_at, + last_seen_at, + created_at, + updated_at +) +VALUES ( + '00000000-0000-0000-0000-000000000801', + 'ASSET-EXISTING', + 'AA:BB:CC:DD:EE:01', + '192.0.2.81', + 'MINER-SN-EXISTING', + 'PSU-SN-EXISTING', + 'online', + '2026-01-01 00:00:00+00', + '2026-01-02 00:00:00+00', + '2025-12-31 00:00:00+00', + '2026-01-02 00:00:00+00' +); + +INSERT INTO miners (id, asset_tag, status) +VALUES + ('00000000-0000-0000-0000-000000000810', 'CaseAsset', 'discovered'), + ('00000000-0000-0000-0000-000000000811', 'caseasset', 'discovered'), + ('00000000-0000-0000-0000-000000000812', 'SpaceAsset', 'discovered'), + ('00000000-0000-0000-0000-000000000813', ' SpaceAsset ', 'discovered'); +SQL + +echo "Checking duplicate Asset ID preflight failure..." +set +e +migration_failure_output="$(run_alembic upgrade head 2>&1)" +migration_failure_status=$? +set -e +if (( migration_failure_status == 0 )); then + echo "Migration unexpectedly accepted conflicting existing Asset IDs." >&2 + exit 1 +fi +for expected_text in \ + "Cannot enforce globally unique Asset IDs" \ + "caseasset" \ + "spaceasset" \ + "Correct the conflicting miners.asset_tag values before upgrading"; do + if [[ "${migration_failure_output}" != *"${expected_text}"* ]]; then + echo "Migration failure did not include expected text: ${expected_text}" >&2 + echo "${migration_failure_output}" >&2 + exit 1 + fi +done + +"${compose[@]}" exec -T postgres sh -ec \ + 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL' +DO $$ +BEGIN + IF (SELECT version_num FROM alembic_version) <> '0007_identity_and_roles' THEN + RAISE EXCEPTION 'Failed migration changed the Alembic revision'; + END IF; + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'miners' + AND column_name IN ('control_board_sn', 'lifecycle_status') + ) OR to_regclass('public.tags') IS NOT NULL + OR to_regclass('public.miner_tags') IS NOT NULL THEN + RAISE EXCEPTION 'Failed migration left partial inventory schema behind'; + END IF; +END +$$; + +DELETE FROM miners +WHERE id IN ( + '00000000-0000-0000-0000-000000000810', + '00000000-0000-0000-0000-000000000811', + '00000000-0000-0000-0000-000000000812', + '00000000-0000-0000-0000-000000000813' +); +SQL + +echo "Upgrading the corrected populated schema to head..." +run_alembic upgrade head + +echo "Checking constraints, defaults, data preservation, and foreign keys..." +"${compose[@]}" exec -T postgres sh -ec \ + 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL' +DO $$ +DECLARE + v_tag_id UUID; + v_cascade_tag_id UUID; +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM miners + WHERE id = '00000000-0000-0000-0000-000000000801' + AND asset_tag = 'ASSET-EXISTING' + AND power_sn = 'PSU-SN-EXISTING' + AND status = 'online' + AND lifecycle_status = 'active' + AND first_seen_at = '2026-01-01 00:00:00+00' + AND last_seen_at = '2026-01-02 00:00:00+00' + ) THEN + RAISE EXCEPTION 'Existing miner data was not preserved'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'miners' + AND column_name IN ('first_seen_at', 'last_seen_at') + AND (is_nullable <> 'YES' OR column_default IS NULL) + ) THEN + RAISE EXCEPTION 'Miner seen timestamps should be nullable and retain defaults'; + END IF; + + BEGIN + INSERT INTO miners (asset_tag, status, lifecycle_status) + VALUES (' asset-existing ', 'manual', 'active'); + RAISE EXCEPTION 'Case-insensitive trimmed duplicate Asset ID was accepted'; + EXCEPTION WHEN unique_violation THEN + NULL; + END; + + INSERT INTO miners ( + id, asset_tag, status, first_seen_at, last_seen_at, created_at + ) VALUES ( + '00000000-0000-0000-0000-000000000803', + NULL, + 'manual', + NULL, + NULL, + '2026-02-03 04:05:06+00' + ); + INSERT INTO miners (id, asset_tag, status) + VALUES ( + '00000000-0000-0000-0000-000000000804', + 'ASSET-DEFAULT-SEEN', + 'manual' + ); + INSERT INTO miners (id, asset_tag, status, lifecycle_status) + VALUES + ('00000000-0000-0000-0000-000000000805', 'ASSET-SPARE', 'manual', 'spare'), + ('00000000-0000-0000-0000-000000000806', 'ASSET-RETIRED', 'manual', 'retired'); + INSERT INTO miners (asset_tag, status, first_seen_at, last_seen_at) + VALUES + (NULL, 'manual', NULL, NULL), + ('', 'manual', NULL, NULL), + (' ', 'manual', NULL, NULL); + + IF NOT EXISTS ( + SELECT 1 FROM miners + WHERE id = '00000000-0000-0000-0000-000000000803' + AND first_seen_at IS NULL + AND last_seen_at IS NULL + AND lifecycle_status = 'active' + ) THEN + RAISE EXCEPTION 'Explicit NULL seen timestamps or lifecycle default were not preserved'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM miners + WHERE id = '00000000-0000-0000-0000-000000000804' + AND first_seen_at IS NOT NULL + AND last_seen_at IS NOT NULL + AND lifecycle_status = 'active' + ) THEN + RAISE EXCEPTION 'Omitted seen timestamps did not receive defaults'; + END IF; + + BEGIN + INSERT INTO miners (asset_tag, status, lifecycle_status) + VALUES ('ASSET-BAD-LIFECYCLE', 'manual', 'maintenance'); + RAISE EXCEPTION 'Invalid lifecycle status was accepted'; + EXCEPTION WHEN check_violation THEN + NULL; + END; + + BEGIN + INSERT INTO tags (name) VALUES (''); + RAISE EXCEPTION 'Blank tag name was accepted'; + EXCEPTION WHEN check_violation THEN + NULL; + END; + BEGIN + INSERT INTO tags (name) VALUES (' '); + RAISE EXCEPTION 'Whitespace-only tag name was accepted'; + EXCEPTION WHEN check_violation THEN + NULL; + END; + + INSERT INTO tags (name) VALUES ('Priority') RETURNING id INTO v_tag_id; + BEGIN + INSERT INTO tags (name) VALUES (' priority '); + RAISE EXCEPTION 'Case-insensitive trimmed duplicate tag was accepted'; + EXCEPTION WHEN unique_violation THEN + NULL; + END; + + INSERT INTO miner_tags (miner_id, tag_id) + VALUES ('00000000-0000-0000-0000-000000000801', v_tag_id); + BEGIN + INSERT INTO miner_tags (miner_id, tag_id) + VALUES ('00000000-0000-0000-0000-000000000801', v_tag_id); + RAISE EXCEPTION 'Duplicate miner-tag association was accepted'; + EXCEPTION WHEN unique_violation THEN + NULL; + END; + + BEGIN + INSERT INTO miner_tags (miner_id, tag_id) + VALUES ('00000000-0000-0000-0000-000000000899', v_tag_id); + RAISE EXCEPTION 'Invalid miner foreign key was accepted'; + EXCEPTION WHEN foreign_key_violation THEN + NULL; + END; + BEGIN + INSERT INTO miner_tags (miner_id, tag_id) + VALUES ( + '00000000-0000-0000-0000-000000000801', + '00000000-0000-0000-0000-000000000899' + ); + RAISE EXCEPTION 'Invalid tag foreign key was accepted'; + EXCEPTION WHEN foreign_key_violation THEN + NULL; + END; + + DELETE FROM tags WHERE id = v_tag_id; + IF EXISTS (SELECT 1 FROM miner_tags WHERE tag_id = v_tag_id) THEN + RAISE EXCEPTION 'Deleting a tag did not cascade to miner_tags'; + END IF; + + INSERT INTO tags (name) VALUES ('Cascade') RETURNING id INTO v_cascade_tag_id; + INSERT INTO miners (id, asset_tag, status) + VALUES ('00000000-0000-0000-0000-000000000802', 'ASSET-CASCADE', 'manual'); + INSERT INTO miner_tags (miner_id, tag_id) + VALUES ('00000000-0000-0000-0000-000000000802', v_cascade_tag_id); + DELETE FROM miners WHERE id = '00000000-0000-0000-0000-000000000802'; + IF EXISTS ( + SELECT 1 FROM miner_tags + WHERE miner_id = '00000000-0000-0000-0000-000000000802' + ) THEN + RAISE EXCEPTION 'Deleting a miner did not cascade to miner_tags'; + END IF; +END +$$; +SQL + +echo "Downgrading to the current schema..." +run_alembic downgrade 0007_identity_and_roles + +echo "Checking downgrade and deterministic timestamp backfill..." +"${compose[@]}" exec -T postgres sh -ec \ + 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL' +DO $$ +BEGIN + IF to_regclass('public.tags') IS NOT NULL + OR to_regclass('public.miner_tags') IS NOT NULL THEN + RAISE EXCEPTION 'Inventory tag tables remain after downgrade'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'miners' + AND column_name IN ('control_board_sn', 'lifecycle_status') + ) THEN + RAISE EXCEPTION 'Inventory miner columns remain after downgrade'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'miners' + AND column_name IN ('first_seen_at', 'last_seen_at') + AND is_nullable <> 'NO' + ) THEN + RAISE EXCEPTION 'Miner seen timestamps were not restored to NOT NULL'; + END IF; + + IF EXISTS ( + SELECT 1 FROM miners + WHERE first_seen_at IS NULL OR last_seen_at IS NULL + ) THEN + RAISE EXCEPTION 'Null seen timestamps were not backfilled'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM miners + WHERE id = '00000000-0000-0000-0000-000000000803' + AND first_seen_at = '2026-02-03 04:05:06+00' + AND last_seen_at = '2026-02-03 04:05:06+00' + ) THEN + RAISE EXCEPTION 'Null seen timestamps were not deterministically backfilled from created_at'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM miners + WHERE id = '00000000-0000-0000-0000-000000000801' + AND asset_tag = 'ASSET-EXISTING' + AND power_sn = 'PSU-SN-EXISTING' + AND status = 'online' + ) THEN + RAISE EXCEPTION 'Existing miner data was not preserved by downgrade'; + END IF; +END +$$; +SQL + +echo "Re-upgrading the populated schema..." +run_alembic upgrade head +populated_revision="$( + "${compose[@]}" exec -T postgres sh -ec \ + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT version_num FROM alembic_version"' +)" +[[ "${populated_revision}" == "0008_manual_miner_inventory" ]] + +echo "Testing an empty database migration from base through head..." +"${compose[@]}" exec -T postgres sh -ec \ + 'createdb -U "$POSTGRES_USER" minerops_inventory_empty' +run_empty_alembic upgrade head +empty_revision="$( + "${compose[@]}" exec -T postgres sh -ec \ + 'psql -U "$POSTGRES_USER" -d minerops_inventory_empty -Atc "SELECT version_num FROM alembic_version"' +)" +[[ "${empty_revision}" == "0008_manual_miner_inventory" ]] + +echo "Manual inventory migration tests passed." diff --git a/sql/verify-development-schema.sql b/sql/verify-development-schema.sql index 19c95af..79f5010 100644 --- a/sql/verify-development-schema.sql +++ b/sql/verify-development-schema.sql @@ -4,6 +4,8 @@ WITH expected(table_name) AS ( ('permissions'), ('users'), ('miners'), + ('tags'), + ('miner_tags'), ('miner_status_snapshots'), ('miner_status_hourly_rollups'), ('miner_status_daily_rollups'),