Prepare legacy migration for the portal cutover

- detection from legacy category only for scores >= 0.5, without ''/none
- category_label + view detection_labeled unify ICU and legacy labels
- schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images
  (same imei, same size, <= 10 min), archived in image_superseded
- migrate_legacy --plan: read-only preview of the migration
- Gundi S3 prefix icu/gundi-test/ -> icu/gundi/

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-16 21:01:17 +02:00
co-authored by Claude Opus 5.5
parent e7e5b6fb87
commit ed105fbff8
9 changed files with 307 additions and 22 deletions
+103 -5
View File
@@ -4,8 +4,11 @@
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
python -m tools.migrate_legacy --verify-only
python -m tools.migrate_legacy --plan # nur lesen: was wuerde migriert/entfernt
python -m tools.migrate_legacy --dedupe --dry-run
python -m tools.migrate_legacy --dedupe # Gundi-Zwillinge entfernen (schema/003)
Wiederholbar. Jeder Batch ist eine eigene Transaktion.
Wiederholbar. Jeder Batch ist eine eigene Transaktion, --dedupe laeuft in einer.
"""
import argparse
import os
@@ -20,8 +23,8 @@ SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
SCHEMA_LOCK_KEY = 7_311_000_001
def load_steps() -> list[tuple[str, str]]:
sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8")
def load_steps(filename: str = "002_migrate_legacy.sql") -> list[tuple[str, str]]:
sql = (SCHEMA_DIR / filename).read_text(encoding="utf-8")
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
@@ -74,6 +77,81 @@ def migrate(conn: psycopg.Connection, batch_size: int) -> None:
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
# Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge).
PLAN_QUERIES = [
("image(entrance) neu",
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
"AND NOT EXISTS (SELECT 1 FROM remote_cam.image i WHERE i.image_uuid = m.metadata_uuid::uuid)"),
("camera_status(entrance) neu",
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
"AND NOT EXISTS (SELECT 1 FROM remote_cam.camera_status s WHERE s.source = 'entrance' "
"AND s.source_ref = m.metadata_uuid)"),
("detection(entrance) neu",
"SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid "
"WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
"AND c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND c.score >= 0.5 "
"AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid "
"AND d.source = 'entrance')"),
("Gundi-Zwillinge (nach Migration zu entfernen)",
"""
WITH pairs AS (
SELECT g.image_uuid AS gundi_uuid, m.metadata_uuid AS legacy_uuid
FROM remote_cam.image g
JOIN remote_cam.metadata m
ON m.imei = g.imei AND m."SizeInByte" = g.size_bytes
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes'
WHERE g.source = 'gundi')
SELECT count(*) FROM (
SELECT count(*) OVER (PARTITION BY gundi_uuid) AS n_g, count(*) OVER (PARTITION BY legacy_uuid) AS n_l
FROM pairs) x
WHERE n_g = 1 AND n_l = 1
"""),
("Gundi-Bilder ohne Zwilling (bleiben)",
"""
SELECT count(*) FROM remote_cam.image g WHERE g.source = 'gundi' AND NOT EXISTS (
SELECT 1 FROM remote_cam.metadata m
WHERE m.imei = g.imei AND m."SizeInByte" = g.size_bytes
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes')
"""),
]
def plan(conn: psycopg.Connection) -> None:
with conn.cursor() as cur:
cur.execute("SET TRANSACTION READ ONLY")
for label, sql in PLAN_QUERIES:
cur.execute(sql)
print(f" {label}: {cur.fetchone()[0]}", flush=True)
conn.rollback()
def dedupe(conn: psycopg.Connection, dry_run: bool) -> None:
steps = dict(load_steps("003_dedupe_gundi_twins.sql"))
with conn.cursor() as cur:
cur.execute(steps["twins"])
cur.execute("SELECT count(*) FROM _twins")
n = cur.fetchone()[0]
cur.execute(
"SELECT min(i.servertime), max(i.servertime) FROM _twins t "
"JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid"
)
lo, hi = cur.fetchone()
print(f" Gundi-Zwillinge: {n} (servertime {lo} .. {hi})", flush=True)
if dry_run:
conn.rollback()
print(" Trockenlauf: nichts geaendert.", flush=True)
return
cur.execute(steps["archive"])
archived = cur.rowcount
cur.execute(steps["delete"])
deleted = cur.rowcount
if deleted != n:
conn.rollback()
raise RuntimeError(f"geloescht={deleted} != Zwillinge={n}; zurueckgerollt")
conn.commit()
print(f" archiviert={archived} geloescht={deleted}", flush=True)
def verify(conn: psycopg.Connection) -> bool:
checks = [
("metadata == image(entrance)",
@@ -82,9 +160,16 @@ def verify(conn: psycopg.Connection) -> bool:
("metadata == camera_status(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
("category == detection(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.category), "
("category (Score >= 0.5, ohne ''/none) == detection(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m "
"ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
"AND c.category NOT IN ('', 'none') AND c.score >= 0.5), "
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
("keine Gundi-Bilder mit Legacy-Zwilling",
"SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e "
"ON e.source = 'entrance' AND e.imei = g.imei AND e.size_bytes = g.size_bytes "
"AND e.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes' "
"WHERE g.source = 'gundi'"),
("metadata ohne ocr_ts == image FAIL",
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
@@ -124,12 +209,25 @@ def main() -> int:
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
ap.add_argument("--batch", type=int, default=20000)
ap.add_argument("--verify-only", action="store_true")
ap.add_argument("--plan", action="store_true", help="nur lesen: Vorschau der Mengen")
ap.add_argument("--dedupe", action="store_true", help="Gundi-Zwillinge entfernen (schema/003)")
ap.add_argument("--dry-run", action="store_true", help="mit --dedupe: nur zaehlen")
args = ap.parse_args()
if not args.dsn:
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
return 2
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
if args.plan:
print("Vorschau (nur lesend):", flush=True)
plan(conn)
return 0
if args.dedupe:
if not args.dry_run:
apply_schema(conn)
print("Gundi-Zwillinge:", flush=True)
dedupe(conn, args.dry_run)
return 0
if not args.verify_only:
apply_schema(conn)
print("Schema angewendet. Migriere ...", flush=True)