- detection from legacy category only for scores >= 0.5, without ''/none - category_label + view detection_labeled unify ICU and legacy labels - schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images (same imei, same size, <= 10 min), archived in image_superseded - migrate_legacy --plan: read-only preview of the migration - Gundi S3 prefix icu/gundi-test/ -> icu/gundi/ Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
241 lines
10 KiB
Python
241 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection).
|
|
|
|
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
|
|
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
|
|
python -m tools.migrate_legacy --verify-only
|
|
python -m tools.migrate_legacy --plan # nur lesen: was wuerde migriert/entfernt
|
|
python -m tools.migrate_legacy --dedupe --dry-run
|
|
python -m tools.migrate_legacy --dedupe # Gundi-Zwillinge entfernen (schema/003)
|
|
|
|
Wiederholbar. Jeder Batch ist eine eigene Transaktion, --dedupe laeuft in einer.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import psycopg
|
|
|
|
SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
|
|
SCHEMA_LOCK_KEY = 7_311_000_001
|
|
|
|
|
|
def load_steps(filename: str = "002_migrate_legacy.sql") -> list[tuple[str, str]]:
|
|
sql = (SCHEMA_DIR / filename).read_text(encoding="utf-8")
|
|
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
|
|
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
|
|
|
|
|
|
def apply_schema(conn: psycopg.Connection) -> None:
|
|
sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8")
|
|
with conn.transaction(), conn.cursor() as cur:
|
|
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
|
|
cur.execute(sql)
|
|
|
|
|
|
def migrate(conn: psycopg.Connection, batch_size: int) -> None:
|
|
steps = load_steps()
|
|
totals = {name: 0 for name, _ in steps}
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT count(*) FROM remote_cam.metadata")
|
|
total_rows = cur.fetchone()[0]
|
|
cur.execute(
|
|
"CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS"
|
|
)
|
|
conn.commit()
|
|
|
|
last = ""
|
|
seen = 0
|
|
t0 = time.perf_counter()
|
|
while True:
|
|
with conn.transaction(), conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO _mig_batch
|
|
SELECT metadata_uuid FROM remote_cam.metadata
|
|
WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s
|
|
""",
|
|
(last, batch_size),
|
|
)
|
|
n = cur.rowcount
|
|
if n == 0:
|
|
break
|
|
cur.execute("SELECT max(metadata_uuid) FROM _mig_batch")
|
|
last = cur.fetchone()[0]
|
|
added = []
|
|
for name, sql in steps:
|
|
cur.execute(sql)
|
|
totals[name] += cur.rowcount
|
|
added.append(f"{name}+{cur.rowcount}")
|
|
seen += n
|
|
rate = seen / max(time.perf_counter() - t0, 1e-6)
|
|
print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True)
|
|
|
|
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
|
|
|
|
|
|
# Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge).
|
|
PLAN_QUERIES = [
|
|
("image(entrance) neu",
|
|
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
|
"AND NOT EXISTS (SELECT 1 FROM remote_cam.image i WHERE i.image_uuid = m.metadata_uuid::uuid)"),
|
|
("camera_status(entrance) neu",
|
|
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
|
"AND NOT EXISTS (SELECT 1 FROM remote_cam.camera_status s WHERE s.source = 'entrance' "
|
|
"AND s.source_ref = m.metadata_uuid)"),
|
|
("detection(entrance) neu",
|
|
"SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid "
|
|
"WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
|
"AND c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND c.score >= 0.5 "
|
|
"AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid "
|
|
"AND d.source = 'entrance')"),
|
|
("Gundi-Zwillinge (nach Migration zu entfernen)",
|
|
"""
|
|
WITH pairs AS (
|
|
SELECT g.image_uuid AS gundi_uuid, m.metadata_uuid AS legacy_uuid
|
|
FROM remote_cam.image g
|
|
JOIN remote_cam.metadata m
|
|
ON m.imei = g.imei AND m."SizeInByte" = g.size_bytes
|
|
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes'
|
|
WHERE g.source = 'gundi')
|
|
SELECT count(*) FROM (
|
|
SELECT count(*) OVER (PARTITION BY gundi_uuid) AS n_g, count(*) OVER (PARTITION BY legacy_uuid) AS n_l
|
|
FROM pairs) x
|
|
WHERE n_g = 1 AND n_l = 1
|
|
"""),
|
|
("Gundi-Bilder ohne Zwilling (bleiben)",
|
|
"""
|
|
SELECT count(*) FROM remote_cam.image g WHERE g.source = 'gundi' AND NOT EXISTS (
|
|
SELECT 1 FROM remote_cam.metadata m
|
|
WHERE m.imei = g.imei AND m."SizeInByte" = g.size_bytes
|
|
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes')
|
|
"""),
|
|
]
|
|
|
|
|
|
def plan(conn: psycopg.Connection) -> None:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SET TRANSACTION READ ONLY")
|
|
for label, sql in PLAN_QUERIES:
|
|
cur.execute(sql)
|
|
print(f" {label}: {cur.fetchone()[0]}", flush=True)
|
|
conn.rollback()
|
|
|
|
|
|
def dedupe(conn: psycopg.Connection, dry_run: bool) -> None:
|
|
steps = dict(load_steps("003_dedupe_gundi_twins.sql"))
|
|
with conn.cursor() as cur:
|
|
cur.execute(steps["twins"])
|
|
cur.execute("SELECT count(*) FROM _twins")
|
|
n = cur.fetchone()[0]
|
|
cur.execute(
|
|
"SELECT min(i.servertime), max(i.servertime) FROM _twins t "
|
|
"JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid"
|
|
)
|
|
lo, hi = cur.fetchone()
|
|
print(f" Gundi-Zwillinge: {n} (servertime {lo} .. {hi})", flush=True)
|
|
if dry_run:
|
|
conn.rollback()
|
|
print(" Trockenlauf: nichts geaendert.", flush=True)
|
|
return
|
|
cur.execute(steps["archive"])
|
|
archived = cur.rowcount
|
|
cur.execute(steps["delete"])
|
|
deleted = cur.rowcount
|
|
if deleted != n:
|
|
conn.rollback()
|
|
raise RuntimeError(f"geloescht={deleted} != Zwillinge={n}; zurueckgerollt")
|
|
conn.commit()
|
|
print(f" archiviert={archived} geloescht={deleted}", flush=True)
|
|
|
|
|
|
def verify(conn: psycopg.Connection) -> bool:
|
|
checks = [
|
|
("metadata == image(entrance)",
|
|
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
|
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"),
|
|
("metadata == camera_status(entrance)",
|
|
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
|
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
|
|
("category (Score >= 0.5, ohne ''/none) == detection(entrance)",
|
|
"SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m "
|
|
"ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
|
"AND c.category NOT IN ('', 'none') AND c.score >= 0.5), "
|
|
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
|
|
("keine Gundi-Bilder mit Legacy-Zwilling",
|
|
"SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e "
|
|
"ON e.source = 'entrance' AND e.imei = g.imei AND e.size_bytes = g.size_bytes "
|
|
"AND e.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes' "
|
|
"WHERE g.source = 'gundi'"),
|
|
("metadata ohne ocr_ts == image FAIL",
|
|
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
|
|
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
|
|
("keine 0/0-Koordinaten (image, camera_status)",
|
|
"SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + "
|
|
"(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"),
|
|
("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)",
|
|
"""
|
|
SELECT 0, count(*) FROM (
|
|
SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200
|
|
) m
|
|
JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid
|
|
WHERE v.status_reported_at IS DISTINCT FROM m.servertime
|
|
OR v.battery_pct IS DISTINCT FROM m."BatteryStatus"
|
|
OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion"
|
|
OR v.status_camera_name IS DISTINCT FROM m."CameraName"
|
|
OR v.signal_strength IS DISTINCT FROM m."SignalStrength"
|
|
OR v.temperature_c IS DISTINCT FROM m."Temperature"
|
|
OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone"
|
|
OR v.ocr_ts IS DISTINCT FROM m.ocr_ts
|
|
"""),
|
|
]
|
|
ok_all = True
|
|
with conn.cursor() as cur:
|
|
for label, sql in checks:
|
|
cur.execute(sql)
|
|
expected, actual = cur.fetchone()
|
|
ok = expected == actual
|
|
ok_all &= ok
|
|
print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True)
|
|
conn.rollback()
|
|
return ok_all
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
|
|
ap.add_argument("--batch", type=int, default=20000)
|
|
ap.add_argument("--verify-only", action="store_true")
|
|
ap.add_argument("--plan", action="store_true", help="nur lesen: Vorschau der Mengen")
|
|
ap.add_argument("--dedupe", action="store_true", help="Gundi-Zwillinge entfernen (schema/003)")
|
|
ap.add_argument("--dry-run", action="store_true", help="mit --dedupe: nur zaehlen")
|
|
args = ap.parse_args()
|
|
if not args.dsn:
|
|
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
|
|
return 2
|
|
|
|
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
|
|
if args.plan:
|
|
print("Vorschau (nur lesend):", flush=True)
|
|
plan(conn)
|
|
return 0
|
|
if args.dedupe:
|
|
if not args.dry_run:
|
|
apply_schema(conn)
|
|
print("Gundi-Zwillinge:", flush=True)
|
|
dedupe(conn, args.dry_run)
|
|
return 0
|
|
if not args.verify_only:
|
|
apply_schema(conn)
|
|
print("Schema angewendet. Migriere ...", flush=True)
|
|
migrate(conn, args.batch)
|
|
print("Verifikation:", flush=True)
|
|
return 0 if verify(conn) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|