Add manual miner inventory schema

backend/alembic/versions/0008_manual_miner_inventory.py

@@ -0,0 +1,193 @@
+"""add manual miner inventory schema
+
+Revision ID: 0008_manual_miner_inventory
+Revises: 0007_identity_and_roles
+Create Date: 2026-07-28
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+
+revision: str = "0008_manual_miner_inventory"
+down_revision: Union[str, None] = "0007_identity_and_roles"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+TAG_TRIMMED_NAME_SQL = (
+    "regexp_replace(name, '^[[:space:]]+|[[:space:]]+$', '', 'g')"
+)
+
+
+def upgrade() -> None:
+    op.add_column("miners", sa.Column("control_board_sn", sa.Text(), nullable=True))
+    op.add_column(
+        "miners",
+        sa.Column(
+            "lifecycle_status",
+            sa.Text(),
+            nullable=False,
+            server_default=sa.text("'active'"),
+        ),
+    )
+    op.create_check_constraint(
+        "ck_miners_lifecycle_status",
+        "miners",
+        "lifecycle_status IN ('active', 'spare', 'retired')",
+    )
+
+    op.execute(
+        """
+        DO $$
+        DECLARE
+            conflicting_asset_ids TEXT;
+        BEGIN
+            SELECT string_agg(
+                format('%L (%s records)', normalized_asset_id, record_count),
+                ', ' ORDER BY normalized_asset_id
+            )
+            INTO conflicting_asset_ids
+            FROM (
+                SELECT
+                    lower(trim(asset_tag)) AS normalized_asset_id,
+                    COUNT(*) AS record_count
+                FROM miners
+                WHERE asset_tag IS NOT NULL
+                  AND trim(asset_tag) <> ''
+                GROUP BY lower(trim(asset_tag))
+                HAVING COUNT(*) > 1
+            ) conflicts;
+
+            IF conflicting_asset_ids IS NOT NULL THEN
+                RAISE EXCEPTION USING
+                    MESSAGE = 'Cannot enforce globally unique Asset IDs. '
+                        || 'Conflicting normalized values: '
+                        || conflicting_asset_ids,
+                    HINT = 'Correct the conflicting miners.asset_tag values '
+                        || 'before upgrading, then run the migration again.';
+            END IF;
+        END
+        $$;
+        """
+    )
+    op.create_index(
+        "uq_miners_asset_tag_ci",
+        "miners",
+        [sa.text("lower(trim(asset_tag))")],
+        unique=True,
+        postgresql_where=sa.text("asset_tag IS NOT NULL AND trim(asset_tag) <> ''"),
+    )
+
+    op.create_table(
+        "tags",
+        sa.Column(
+            "id",
+            postgresql.UUID(as_uuid=True),
+            primary_key=True,
+            server_default=sa.text("gen_random_uuid()"),
+        ),
+        sa.Column("name", sa.Text(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.CheckConstraint(
+            f"{TAG_TRIMMED_NAME_SQL} <> ''",
+            name="ck_tags_name_not_blank",
+        ),
+    )
+    op.create_index(
+        "uq_tags_name_ci",
+        "tags",
+        [sa.text(f"lower({TAG_TRIMMED_NAME_SQL})")],
+        unique=True,
+    )
+
+    op.create_table(
+        "miner_tags",
+        sa.Column(
+            "miner_id",
+            postgresql.UUID(as_uuid=True),
+            sa.ForeignKey("miners.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "tag_id",
+            postgresql.UUID(as_uuid=True),
+            sa.ForeignKey("tags.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.PrimaryKeyConstraint("miner_id", "tag_id", name="pk_miner_tags"),
+    )
+    op.create_index("idx_miner_tags_tag_id", "miner_tags", ["tag_id"])
+
+    # Discovery continues to receive now() defaults. Manual creation can
+    # explicitly store NULL to represent an asset that has never been observed.
+    op.alter_column(
+        "miners",
+        "first_seen_at",
+        existing_type=sa.DateTime(timezone=True),
+        existing_server_default=sa.text("now()"),
+        nullable=True,
+    )
+    op.alter_column(
+        "miners",
+        "last_seen_at",
+        existing_type=sa.DateTime(timezone=True),
+        existing_server_default=sa.text("now()"),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    # created_at is non-null in the pre-0008 schema, making this deterministic.
+    op.execute(
+        """
+        UPDATE miners
+        SET first_seen_at = COALESCE(first_seen_at, created_at),
+            last_seen_at = COALESCE(last_seen_at, created_at)
+        WHERE first_seen_at IS NULL
+           OR last_seen_at IS NULL;
+        """
+    )
+    op.alter_column(
+        "miners",
+        "last_seen_at",
+        existing_type=sa.DateTime(timezone=True),
+        existing_server_default=sa.text("now()"),
+        nullable=False,
+    )
+    op.alter_column(
+        "miners",
+        "first_seen_at",
+        existing_type=sa.DateTime(timezone=True),
+        existing_server_default=sa.text("now()"),
+        nullable=False,
+    )
+
+    op.drop_index("idx_miner_tags_tag_id", table_name="miner_tags")
+    op.drop_table("miner_tags")
+    op.drop_index("uq_tags_name_ci", table_name="tags")
+    op.drop_table("tags")
+
+    op.drop_index("uq_miners_asset_tag_ci", table_name="miners")
+    op.drop_constraint("ck_miners_lifecycle_status", "miners", type_="check")
+    op.drop_column("miners", "lifecycle_status")
+    op.drop_column("miners", "control_board_sn")

scripts/test-dev-baseline.sh

@@ -88,7 +88,7 @@ health_json="$(cat "${health_file}")"
 
 echo "Checking migrations and required schema..."
 current_revision="$("${compose[@]}" exec -T backend alembic current 2>/dev/null)"
-[[ "${current_revision}" == *"0007_identity_and_roles"* ]]
+[[ "${current_revision}" == *"0008_manual_miner_inventory"* ]]
 
 missing_tables="$(
     "${compose[@]}" exec -T postgres sh -c \

scripts/test-manual-inventory-migration.sh

@@ -0,0 +1,398 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+project_name="${MINEROPS_INVENTORY_MIGRATION_PROJECT:-minerops-inventory-migration-$$}"
+compose=(
+    docker compose
+    --project-name "${project_name}"
+    --env-file .env.dev
+    -f compose.dev.yaml
+)
+empty_database="minerops_inventory_empty"
+
+cleanup() {
+    status=$?
+    if (( status != 0 )); then
+        echo
+        echo "Migration test failed. Recent PostgreSQL logs:"
+        "${compose[@]}" logs --no-color --tail 120 postgres || true
+    fi
+    "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
+    exit "${status}"
+}
+
+trap cleanup EXIT
+
+if [[ ! -f .env.dev ]]; then
+    echo "Missing .env.dev. Run ./scripts/dev-env-init.sh first." >&2
+    exit 1
+fi
+
+run_alembic() {
+    "${compose[@]}" run --rm --no-deps backend alembic "$@"
+}
+
+run_empty_alembic() {
+    "${compose[@]}" run --rm --no-deps \
+        -e "POSTGRES_DB=${empty_database}" \
+        backend alembic "$@"
+}
+
+echo "Starting disposable PostgreSQL..."
+"${compose[@]}" up --detach postgres
+for attempt in $(seq 1 60); do
+    if "${compose[@]}" exec -T postgres sh -ec \
+        'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null 2>&1; then
+        break
+    fi
+    if (( attempt == 60 )); then
+        echo "PostgreSQL did not become ready." >&2
+        exit 1
+    fi
+    sleep 1
+done
+
+echo "Building migration image..."
+"${compose[@]}" build backend
+
+echo "Upgrading to the pre-inventory schema..."
+run_alembic upgrade 0007_identity_and_roles
+
+echo "Seeding existing miner data and conflicting Asset IDs..."
+"${compose[@]}" exec -T postgres sh -ec \
+    'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL'
+INSERT INTO miners (
+    id,
+    asset_tag,
+    current_mac,
+    current_ip,
+    miner_sn,
+    power_sn,
+    status,
+    first_seen_at,
+    last_seen_at,
+    created_at,
+    updated_at
+)
+VALUES (
+    '00000000-0000-0000-0000-000000000801',
+    'ASSET-EXISTING',
+    'AA:BB:CC:DD:EE:01',
+    '192.0.2.81',
+    'MINER-SN-EXISTING',
+    'PSU-SN-EXISTING',
+    'online',
+    '2026-01-01 00:00:00+00',
+    '2026-01-02 00:00:00+00',
+    '2025-12-31 00:00:00+00',
+    '2026-01-02 00:00:00+00'
+);
+
+INSERT INTO miners (id, asset_tag, status)
+VALUES
+    ('00000000-0000-0000-0000-000000000810', 'CaseAsset', 'discovered'),
+    ('00000000-0000-0000-0000-000000000811', 'caseasset', 'discovered'),
+    ('00000000-0000-0000-0000-000000000812', 'SpaceAsset', 'discovered'),
+    ('00000000-0000-0000-0000-000000000813', '  SpaceAsset  ', 'discovered');
+SQL
+
+echo "Checking duplicate Asset ID preflight failure..."
+set +e
+migration_failure_output="$(run_alembic upgrade head 2>&1)"
+migration_failure_status=$?
+set -e
+if (( migration_failure_status == 0 )); then
+    echo "Migration unexpectedly accepted conflicting existing Asset IDs." >&2
+    exit 1
+fi
+for expected_text in \
+    "Cannot enforce globally unique Asset IDs" \
+    "caseasset" \
+    "spaceasset" \
+    "Correct the conflicting miners.asset_tag values before upgrading"; do
+    if [[ "${migration_failure_output}" != *"${expected_text}"* ]]; then
+        echo "Migration failure did not include expected text: ${expected_text}" >&2
+        echo "${migration_failure_output}" >&2
+        exit 1
+    fi
+done
+
+"${compose[@]}" exec -T postgres sh -ec \
+    'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL'
+DO $$
+BEGIN
+    IF (SELECT version_num FROM alembic_version) <> '0007_identity_and_roles' THEN
+        RAISE EXCEPTION 'Failed migration changed the Alembic revision';
+    END IF;
+    IF EXISTS (
+        SELECT 1 FROM information_schema.columns
+        WHERE table_schema = 'public'
+          AND table_name = 'miners'
+          AND column_name IN ('control_board_sn', 'lifecycle_status')
+    ) OR to_regclass('public.tags') IS NOT NULL
+      OR to_regclass('public.miner_tags') IS NOT NULL THEN
+        RAISE EXCEPTION 'Failed migration left partial inventory schema behind';
+    END IF;
+END
+$$;
+
+DELETE FROM miners
+WHERE id IN (
+    '00000000-0000-0000-0000-000000000810',
+    '00000000-0000-0000-0000-000000000811',
+    '00000000-0000-0000-0000-000000000812',
+    '00000000-0000-0000-0000-000000000813'
+);
+SQL
+
+echo "Upgrading the corrected populated schema to head..."
+run_alembic upgrade head
+
+echo "Checking constraints, defaults, data preservation, and foreign keys..."
+"${compose[@]}" exec -T postgres sh -ec \
+    'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL'
+DO $$
+DECLARE
+    v_tag_id UUID;
+    v_cascade_tag_id UUID;
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM miners
+        WHERE id = '00000000-0000-0000-0000-000000000801'
+          AND asset_tag = 'ASSET-EXISTING'
+          AND power_sn = 'PSU-SN-EXISTING'
+          AND status = 'online'
+          AND lifecycle_status = 'active'
+          AND first_seen_at = '2026-01-01 00:00:00+00'
+          AND last_seen_at = '2026-01-02 00:00:00+00'
+    ) THEN
+        RAISE EXCEPTION 'Existing miner data was not preserved';
+    END IF;
+
+    IF EXISTS (
+        SELECT 1
+        FROM information_schema.columns
+        WHERE table_schema = 'public'
+          AND table_name = 'miners'
+          AND column_name IN ('first_seen_at', 'last_seen_at')
+          AND (is_nullable <> 'YES' OR column_default IS NULL)
+    ) THEN
+        RAISE EXCEPTION 'Miner seen timestamps should be nullable and retain defaults';
+    END IF;
+
+    BEGIN
+        INSERT INTO miners (asset_tag, status, lifecycle_status)
+        VALUES (' asset-existing ', 'manual', 'active');
+        RAISE EXCEPTION 'Case-insensitive trimmed duplicate Asset ID was accepted';
+    EXCEPTION WHEN unique_violation THEN
+        NULL;
+    END;
+
+    INSERT INTO miners (
+        id, asset_tag, status, first_seen_at, last_seen_at, created_at
+    ) VALUES (
+        '00000000-0000-0000-0000-000000000803',
+        NULL,
+        'manual',
+        NULL,
+        NULL,
+        '2026-02-03 04:05:06+00'
+    );
+    INSERT INTO miners (id, asset_tag, status)
+    VALUES (
+        '00000000-0000-0000-0000-000000000804',
+        'ASSET-DEFAULT-SEEN',
+        'manual'
+    );
+    INSERT INTO miners (id, asset_tag, status, lifecycle_status)
+    VALUES
+        ('00000000-0000-0000-0000-000000000805', 'ASSET-SPARE', 'manual', 'spare'),
+        ('00000000-0000-0000-0000-000000000806', 'ASSET-RETIRED', 'manual', 'retired');
+    INSERT INTO miners (asset_tag, status, first_seen_at, last_seen_at)
+    VALUES
+        (NULL, 'manual', NULL, NULL),
+        ('', 'manual', NULL, NULL),
+        ('   ', 'manual', NULL, NULL);
+
+    IF NOT EXISTS (
+        SELECT 1 FROM miners
+        WHERE id = '00000000-0000-0000-0000-000000000803'
+          AND first_seen_at IS NULL
+          AND last_seen_at IS NULL
+          AND lifecycle_status = 'active'
+    ) THEN
+        RAISE EXCEPTION 'Explicit NULL seen timestamps or lifecycle default were not preserved';
+    END IF;
+    IF NOT EXISTS (
+        SELECT 1 FROM miners
+        WHERE id = '00000000-0000-0000-0000-000000000804'
+          AND first_seen_at IS NOT NULL
+          AND last_seen_at IS NOT NULL
+          AND lifecycle_status = 'active'
+    ) THEN
+        RAISE EXCEPTION 'Omitted seen timestamps did not receive defaults';
+    END IF;
+
+    BEGIN
+        INSERT INTO miners (asset_tag, status, lifecycle_status)
+        VALUES ('ASSET-BAD-LIFECYCLE', 'manual', 'maintenance');
+        RAISE EXCEPTION 'Invalid lifecycle status was accepted';
+    EXCEPTION WHEN check_violation THEN
+        NULL;
+    END;
+
+    BEGIN
+        INSERT INTO tags (name) VALUES ('');
+        RAISE EXCEPTION 'Blank tag name was accepted';
+    EXCEPTION WHEN check_violation THEN
+        NULL;
+    END;
+    BEGIN
+        INSERT INTO tags (name) VALUES ('   ');
+        RAISE EXCEPTION 'Whitespace-only tag name was accepted';
+    EXCEPTION WHEN check_violation THEN
+        NULL;
+    END;
+
+    INSERT INTO tags (name) VALUES ('Priority') RETURNING id INTO v_tag_id;
+    BEGIN
+        INSERT INTO tags (name) VALUES (' priority ');
+        RAISE EXCEPTION 'Case-insensitive trimmed duplicate tag was accepted';
+    EXCEPTION WHEN unique_violation THEN
+        NULL;
+    END;
+
+    INSERT INTO miner_tags (miner_id, tag_id)
+    VALUES ('00000000-0000-0000-0000-000000000801', v_tag_id);
+    BEGIN
+        INSERT INTO miner_tags (miner_id, tag_id)
+        VALUES ('00000000-0000-0000-0000-000000000801', v_tag_id);
+        RAISE EXCEPTION 'Duplicate miner-tag association was accepted';
+    EXCEPTION WHEN unique_violation THEN
+        NULL;
+    END;
+
+    BEGIN
+        INSERT INTO miner_tags (miner_id, tag_id)
+        VALUES ('00000000-0000-0000-0000-000000000899', v_tag_id);
+        RAISE EXCEPTION 'Invalid miner foreign key was accepted';
+    EXCEPTION WHEN foreign_key_violation THEN
+        NULL;
+    END;
+    BEGIN
+        INSERT INTO miner_tags (miner_id, tag_id)
+        VALUES (
+            '00000000-0000-0000-0000-000000000801',
+            '00000000-0000-0000-0000-000000000899'
+        );
+        RAISE EXCEPTION 'Invalid tag foreign key was accepted';
+    EXCEPTION WHEN foreign_key_violation THEN
+        NULL;
+    END;
+
+    DELETE FROM tags WHERE id = v_tag_id;
+    IF EXISTS (SELECT 1 FROM miner_tags WHERE tag_id = v_tag_id) THEN
+        RAISE EXCEPTION 'Deleting a tag did not cascade to miner_tags';
+    END IF;
+
+    INSERT INTO tags (name) VALUES ('Cascade') RETURNING id INTO v_cascade_tag_id;
+    INSERT INTO miners (id, asset_tag, status)
+    VALUES ('00000000-0000-0000-0000-000000000802', 'ASSET-CASCADE', 'manual');
+    INSERT INTO miner_tags (miner_id, tag_id)
+    VALUES ('00000000-0000-0000-0000-000000000802', v_cascade_tag_id);
+    DELETE FROM miners WHERE id = '00000000-0000-0000-0000-000000000802';
+    IF EXISTS (
+        SELECT 1 FROM miner_tags
+        WHERE miner_id = '00000000-0000-0000-0000-000000000802'
+    ) THEN
+        RAISE EXCEPTION 'Deleting a miner did not cascade to miner_tags';
+    END IF;
+END
+$$;
+SQL
+
+echo "Downgrading to the current schema..."
+run_alembic downgrade 0007_identity_and_roles
+
+echo "Checking downgrade and deterministic timestamp backfill..."
+"${compose[@]}" exec -T postgres sh -ec \
+    'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL'
+DO $$
+BEGIN
+    IF to_regclass('public.tags') IS NOT NULL
+       OR to_regclass('public.miner_tags') IS NOT NULL THEN
+        RAISE EXCEPTION 'Inventory tag tables remain after downgrade';
+    END IF;
+
+    IF EXISTS (
+        SELECT 1
+        FROM information_schema.columns
+        WHERE table_schema = 'public'
+          AND table_name = 'miners'
+          AND column_name IN ('control_board_sn', 'lifecycle_status')
+    ) THEN
+        RAISE EXCEPTION 'Inventory miner columns remain after downgrade';
+    END IF;
+
+    IF EXISTS (
+        SELECT 1
+        FROM information_schema.columns
+        WHERE table_schema = 'public'
+          AND table_name = 'miners'
+          AND column_name IN ('first_seen_at', 'last_seen_at')
+          AND is_nullable <> 'NO'
+    ) THEN
+        RAISE EXCEPTION 'Miner seen timestamps were not restored to NOT NULL';
+    END IF;
+
+    IF EXISTS (
+        SELECT 1 FROM miners
+        WHERE first_seen_at IS NULL OR last_seen_at IS NULL
+    ) THEN
+        RAISE EXCEPTION 'Null seen timestamps were not backfilled';
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1
+        FROM miners
+        WHERE id = '00000000-0000-0000-0000-000000000803'
+          AND first_seen_at = '2026-02-03 04:05:06+00'
+          AND last_seen_at = '2026-02-03 04:05:06+00'
+    ) THEN
+        RAISE EXCEPTION 'Null seen timestamps were not deterministically backfilled from created_at';
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1
+        FROM miners
+        WHERE id = '00000000-0000-0000-0000-000000000801'
+          AND asset_tag = 'ASSET-EXISTING'
+          AND power_sn = 'PSU-SN-EXISTING'
+          AND status = 'online'
+    ) THEN
+        RAISE EXCEPTION 'Existing miner data was not preserved by downgrade';
+    END IF;
+END
+$$;
+SQL
+
+echo "Re-upgrading the populated schema..."
+run_alembic upgrade head
+populated_revision="$(
+    "${compose[@]}" exec -T postgres sh -ec \
+        'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT version_num FROM alembic_version"'
+)"
+[[ "${populated_revision}" == "0008_manual_miner_inventory" ]]
+
+echo "Testing an empty database migration from base through head..."
+"${compose[@]}" exec -T postgres sh -ec \
+    'createdb -U "$POSTGRES_USER" minerops_inventory_empty'
+run_empty_alembic upgrade head
+empty_revision="$(
+    "${compose[@]}" exec -T postgres sh -ec \
+        'psql -U "$POSTGRES_USER" -d minerops_inventory_empty -Atc "SELECT version_num FROM alembic_version"'
+)"
+[[ "${empty_revision}" == "0008_manual_miner_inventory" ]]
+
+echo "Manual inventory migration tests passed."

sql/verify-development-schema.sql

@@ -4,6 +4,8 @@ WITH expected(table_name) AS (
         ('permissions'),
         ('users'),
         ('miners'),
+        ('tags'),
+        ('miner_tags'),
         ('miner_status_snapshots'),
         ('miner_status_hourly_rollups'),
         ('miner_status_daily_rollups'),