#!/usr/bin/env python3 """Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection). python -m tools.migrate_legacy # PG_DSN aus der Umgebung python -m tools.migrate_legacy --dsn postgresql://... --batch 20000 python -m tools.migrate_legacy --verify-only Wiederholbar. Jeder Batch ist eine eigene Transaktion. """ import argparse import os import re import sys import time from pathlib import Path import psycopg SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema" SCHEMA_LOCK_KEY = 7_311_000_001 def load_steps() -> list[tuple[str, str]]: sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8") parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE) return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)] def apply_schema(conn: psycopg.Connection) -> None: sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8") with conn.transaction(), conn.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,)) cur.execute(sql) def migrate(conn: psycopg.Connection, batch_size: int) -> None: steps = load_steps() totals = {name: 0 for name, _ in steps} with conn.cursor() as cur: cur.execute("SELECT count(*) FROM remote_cam.metadata") total_rows = cur.fetchone()[0] cur.execute( "CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS" ) conn.commit() last = "" seen = 0 t0 = time.perf_counter() while True: with conn.transaction(), conn.cursor() as cur: cur.execute( """ INSERT INTO _mig_batch SELECT metadata_uuid FROM remote_cam.metadata WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s """, (last, batch_size), ) n = cur.rowcount if n == 0: break cur.execute("SELECT max(metadata_uuid) FROM _mig_batch") last = cur.fetchone()[0] added = [] for name, sql in steps: cur.execute(sql) totals[name] += cur.rowcount added.append(f"{name}+{cur.rowcount}") seen += n rate = seen / max(time.perf_counter() - t0, 1e-6) print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True) print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True) def verify(conn: psycopg.Connection) -> bool: checks = [ ("metadata == image(entrance)", "SELECT (SELECT count(*) FROM remote_cam.metadata), " "(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"), ("metadata == camera_status(entrance)", "SELECT (SELECT count(*) FROM remote_cam.metadata), " "(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"), ("category == detection(entrance)", "SELECT (SELECT count(*) FROM remote_cam.category), " "(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"), ("metadata ohne ocr_ts == image FAIL", "SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), " "(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"), ("keine 0/0-Koordinaten (image, camera_status)", "SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + " "(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"), ("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)", """ SELECT 0, count(*) FROM ( SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200 ) m JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid WHERE v.status_reported_at IS DISTINCT FROM m.servertime OR v.battery_pct IS DISTINCT FROM m."BatteryStatus" OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion" OR v.status_camera_name IS DISTINCT FROM m."CameraName" OR v.signal_strength IS DISTINCT FROM m."SignalStrength" OR v.temperature_c IS DISTINCT FROM m."Temperature" OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone" OR v.ocr_ts IS DISTINCT FROM m.ocr_ts """), ] ok_all = True with conn.cursor() as cur: for label, sql in checks: cur.execute(sql) expected, actual = cur.fetchone() ok = expected == actual ok_all &= ok print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True) conn.rollback() return ok_all def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dsn", default=os.environ.get("PG_DSN")) ap.add_argument("--batch", type=int, default=20000) ap.add_argument("--verify-only", action="store_true") args = ap.parse_args() if not args.dsn: print("--dsn oder PG_DSN fehlt", file=sys.stderr) return 2 with psycopg.connect(args.dsn, connect_timeout=10) as conn: if not args.verify_only: apply_schema(conn) print("Schema angewendet. Migriere ...", flush=True) migrate(conn, args.batch) print("Verifikation:", flush=True) return 0 if verify(conn) else 1 if __name__ == "__main__": raise SystemExit(main())