Add Gundi v2 push receiver for ICU and normalized data model

ICU now pushes camera events via a Gundi-v2-compatible API instead of
dropping file triplets into S3. The new gundi-receiver service accepts
them directly, so images are in the system within seconds instead of
waiting out the 90s age gate + poll interval. OCR of the burnt-in camera
timestamp stays mandatory (ICU only sends server times; OCR detects
drifting camera clocks) but now runs asynchronously after ingest.

The push path writes a normalized model that separates what the legacy
metadata table conflates: camera_status (status time series), image
(image properties) and detection (classifications), plus the
image_with_status view as the portal's migration target. A repeatable,
batched migration splits existing metadata/category rows into it.

The legacy poller is unchanged and keeps writing metadata/category until
the portal is switched over.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-09-18 13:54:21 +00:00
co-authored by Claude Opus 5
parent 94bfc60fc4
commit 2fc8dea10c
13 changed files with 1638 additions and 11 deletions
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection).
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
python -m tools.migrate_legacy --verify-only
Wiederholbar. Jeder Batch ist eine eigene Transaktion.
"""
import argparse
import os
import re
import sys
import time
from pathlib import Path
import psycopg
SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
SCHEMA_LOCK_KEY = 7_311_000_001
def load_steps() -> list[tuple[str, str]]:
sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8")
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
def apply_schema(conn: psycopg.Connection) -> None:
sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8")
with conn.transaction(), conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
cur.execute(sql)
def migrate(conn: psycopg.Connection, batch_size: int) -> None:
steps = load_steps()
totals = {name: 0 for name, _ in steps}
with conn.cursor() as cur:
cur.execute("SELECT count(*) FROM remote_cam.metadata")
total_rows = cur.fetchone()[0]
cur.execute(
"CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS"
)
conn.commit()
last = ""
seen = 0
t0 = time.perf_counter()
while True:
with conn.transaction(), conn.cursor() as cur:
cur.execute(
"""
INSERT INTO _mig_batch
SELECT metadata_uuid FROM remote_cam.metadata
WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s
""",
(last, batch_size),
)
n = cur.rowcount
if n == 0:
break
cur.execute("SELECT max(metadata_uuid) FROM _mig_batch")
last = cur.fetchone()[0]
added = []
for name, sql in steps:
cur.execute(sql)
totals[name] += cur.rowcount
added.append(f"{name}+{cur.rowcount}")
seen += n
rate = seen / max(time.perf_counter() - t0, 1e-6)
print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True)
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
def verify(conn: psycopg.Connection) -> bool:
checks = [
("metadata == image(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"),
("metadata == camera_status(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
("category == detection(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.category), "
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
("metadata ohne ocr_ts == image FAIL",
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
("keine 0/0-Koordinaten (image, camera_status)",
"SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + "
"(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"),
("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)",
"""
SELECT 0, count(*) FROM (
SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200
) m
JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid
WHERE v.status_reported_at IS DISTINCT FROM m.servertime
OR v.battery_pct IS DISTINCT FROM m."BatteryStatus"
OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion"
OR v.status_camera_name IS DISTINCT FROM m."CameraName"
OR v.signal_strength IS DISTINCT FROM m."SignalStrength"
OR v.temperature_c IS DISTINCT FROM m."Temperature"
OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone"
OR v.ocr_ts IS DISTINCT FROM m.ocr_ts
"""),
]
ok_all = True
with conn.cursor() as cur:
for label, sql in checks:
cur.execute(sql)
expected, actual = cur.fetchone()
ok = expected == actual
ok_all &= ok
print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True)
conn.rollback()
return ok_all
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
ap.add_argument("--batch", type=int, default=20000)
ap.add_argument("--verify-only", action="store_true")
args = ap.parse_args()
if not args.dsn:
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
return 2
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
if not args.verify_only:
apply_schema(conn)
print("Schema angewendet. Migriere ...", flush=True)
migrate(conn, args.batch)
print("Verifikation:", flush=True)
return 0 if verify(conn) else 1
if __name__ == "__main__":
raise SystemExit(main())