Add Gundi v2 push receiver for ICU and normalized data model

ICU now pushes camera events via a Gundi-v2-compatible API instead of
dropping file triplets into S3. The new gundi-receiver service accepts
them directly, so images are in the system within seconds instead of
waiting out the 90s age gate + poll interval. OCR of the burnt-in camera
timestamp stays mandatory (ICU only sends server times; OCR detects
drifting camera clocks) but now runs asynchronously after ingest.

The push path writes a normalized model that separates what the legacy
metadata table conflates: camera_status (status time series), image
(image properties) and detection (classifications), plus the
image_with_status view as the portal's migration target. A repeatable,
batched migration splits existing metadata/category rows into it.

The legacy poller is unchanged and keeps writing metadata/category until
the portal is switched over.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-09-18 13:54:21 +00:00
co-authored by Claude Opus 5
parent 94bfc60fc4
commit 2fc8dea10c
13 changed files with 1638 additions and 11 deletions
+176
View File
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""Spielt die Beispiel-Payloads aus der ICU-Doku gegen den Gundi-Receiver.
python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg bild.jpg
Ohne --jpeg wird ein synthetisches Bild mit Zeitstempel-Overlay erzeugt (braucht Pillow).
Standard-IMEI ist eine Test-IMEI, damit echte Kamerazeitreihen unberuehrt bleiben.
"""
import argparse
import http.client
import io
import json
import os
import sys
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import urlparse
TEST_IMEI = "999000000000001"
def request(url: str, method: str, path: str, *, body: bytes = b"", headers: dict | None = None):
u = urlparse(url)
cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
conn = cls(u.hostname, u.port, timeout=30)
try:
conn.request(method, path, body=body, headers=headers or {})
resp = conn.getresponse()
data = resp.read()
return resp.status, dict(resp.getheaders()), data
finally:
conn.close()
def post_json(url, path, key, payload):
headers = {"Content-Type": "application/json"}
if key is not None:
headers["apikey"] = key
return request(url, "POST", path, body=json.dumps(payload).encode(), headers=headers)
def post_file(url, path, key, jpg: bytes):
boundary = f"----smoke-{uuid.uuid4().hex}"
body = (
f"--{boundary}\r\n"
f'Content-Disposition: form-data; name="file1"; filename="../../evil.jpg"\r\n'
f"Content-Type: image/jpeg\r\n\r\n"
).encode() + jpg + f"\r\n--{boundary}--\r\n".encode()
return request(url, "POST", path, body=body, headers={
"Content-Type": f"multipart/form-data; boundary={boundary}",
"apikey": key,
})
def synthetic_jpeg(ts: datetime) -> bytes:
from PIL import Image, ImageDraw, ImageFont
w, h = 1920, 1080
im = Image.new("RGB", (w, h), (70, 90, 70))
draw = ImageDraw.Draw(im)
bar = int(h * 0.05)
draw.rectangle((0, h - bar, w, h), fill=(0, 0, 0))
font = ImageFont.load_default(size=int(bar * 0.8))
text = ts.strftime("%H:%M:%S %d-%m-%Y")
tw = draw.textlength(text, font=font)
draw.text((w - tw - 20, h - bar + bar * 0.08), text, fill=(255, 255, 255), font=font)
out = io.BytesIO()
im.save(out, format="JPEG", quality=90)
return out.getvalue()
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--url", default="http://127.0.0.1:8090")
ap.add_argument("--key", default=os.environ.get("GUNDI_API_KEY"))
ap.add_argument("--imei", default=TEST_IMEI)
ap.add_argument("--jpeg", help="Pfad zu einem echten Kamerabild")
args = ap.parse_args()
if not args.key:
print("--key oder GUNDI_API_KEY fehlt", file=sys.stderr)
return 2
failures = 0
def check(label: str, ok: bool, detail: str = "") -> None:
nonlocal failures
failures += 0 if ok else 1
print(f"[{'OK ' if ok else 'FAIL'}] {label}{(' - ' + detail) if detail else ''}")
now = datetime.now(timezone.utc).replace(microsecond=0)
status_payload = {
"source": args.imei,
"event_type": "camera_status_report",
"title": "Camera Status report - Smoke Test",
"recorded_at": (now - timedelta(minutes=10)).isoformat().replace("+00:00", "Z"),
"location": {"lat": 0.0, "lon": 0.0},
"event_details": {
"imei": args.imei, "iccid": "8900000000000000000", "init_key": "000000000000",
"camera_name": "Smoke Test", "camera_model": None, "firmware_version": "1.4.2",
"mobile_country_code": "232", "mobile_network_code": "01", "location_area_code": "1234",
"cell_id": "567890", "customer_timezone": "Europe/Vienna", "battery_status": 87,
"signal_strength": 3, "temperature": -4, "coordinate_switch": False, "work_period": True,
"image_format": "F4_3", "image_night": "MIN", "work_start": "06:00", "work_end": "21:00",
"thumbnail_size": "720p", "pir_interval": "30", "time_scan": "120", "camera_language": "en",
},
}
trap_payload = {
"source": args.imei,
"event_type": "cameratrap_rep",
"title": "Camera trap image - Smoke Test",
"recorded_at": now.isoformat().replace("+00:00", "Z"),
"location": {"lat": 47.8095, "lon": 13.0550},
"event_details": {
"cameratraprep_camera-name": "Smoke Test",
"cameratraprep_camera-make": "M350GW",
"detections": [
{"category": "deer", "confidence_percent": 96.2, "x_px": 0, "y_px": 0, "width_px": 0, "height_px": 0},
{"category": "human", "confidence_percent": 3.1, "x_px": 15, "y_px": 17, "width_px": 100, "height_px": 200},
],
},
}
code, _, _ = request(args.url, "GET", "/healthz")
check("healthz", code == 200, f"HTTP {code}")
code, _, _ = post_json(args.url, "/v2/events/", None, status_payload)
check("ohne apikey -> 401", code == 401, f"HTTP {code}")
code, _, _ = post_json(args.url, "/v2/events/", "falsch", status_payload)
check("falscher apikey -> 401", code == 401, f"HTTP {code}")
code, headers, _ = post_json(args.url, "/v2/events", args.key, status_payload)
check("ohne Slash -> 308", code == 308 and headers.get("Location") == "/v2/events/",
f"HTTP {code} Location={headers.get('Location')}")
code, _, data = post_json(args.url, "/v2/events/", args.key, {"event_type": "cameratrap_rep"})
check("unvollstaendiges Event -> 400", code == 400, f"HTTP {code} {data[:120]!r}")
code, _, data = post_json(args.url, "/v2/events/", args.key, status_payload)
status_id = json.loads(data).get("object_id") if code == 200 else None
check("camera_status_report", code == 200 and bool(status_id), f"HTTP {code} object_id={status_id}")
code, _, _ = post_json(args.url, "/v2/events/", args.key, status_payload)
check("camera_status_report erneut (idempotent in camera_status)", code == 200, f"HTTP {code}")
code, _, data = post_json(args.url, "/v2/events/", args.key, trap_payload)
object_id = json.loads(data).get("object_id") if code == 200 else None
check("cameratrap_rep", code == 200 and bool(object_id), f"HTTP {code} object_id={object_id}")
if args.jpeg:
with open(args.jpeg, "rb") as f:
jpg = f.read()
else:
jpg = synthetic_jpeg(now.astimezone(timezone(timedelta(hours=2))))
code, _, _ = post_file(args.url, f"/v2/events/{uuid.uuid4()}/attachments/", args.key, jpg)
check("Attachment auf unbekannte object_id -> 404", code == 404, f"HTTP {code}")
code, _, _ = post_file(args.url, "/v2/events/..%2F..%2Fx/attachments/", args.key, jpg)
check("Attachment auf ungueltige object_id -> 404", code == 404, f"HTTP {code}")
code, _, _ = post_file(args.url, f"/v2/events/{status_id}/attachments/", args.key, jpg)
check("Attachment auf Status-Event -> 404", code == 404, f"HTTP {code}")
if object_id:
code, headers, _ = post_file(args.url, f"/v2/events/{object_id}/attachments", args.key, jpg)
check("Attachment ohne Slash -> 308", code == 308, f"HTTP {code}")
code, _, data = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg)
check("Attachment", code == 200, f"HTTP {code} {data[:120]!r}")
code, _, _ = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg)
check("Attachment erneut (duplicate, 200)", code == 200, f"HTTP {code}")
print(f"\nobject_id cameratrap_rep: {object_id}")
print(f"object_id status_report: {status_id}")
print(f"{failures} Fehler")
return 1 if failures else 0
if __name__ == "__main__":
raise SystemExit(main())
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection).
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
python -m tools.migrate_legacy --verify-only
Wiederholbar. Jeder Batch ist eine eigene Transaktion.
"""
import argparse
import os
import re
import sys
import time
from pathlib import Path
import psycopg
SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
SCHEMA_LOCK_KEY = 7_311_000_001
def load_steps() -> list[tuple[str, str]]:
sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8")
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
def apply_schema(conn: psycopg.Connection) -> None:
sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8")
with conn.transaction(), conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
cur.execute(sql)
def migrate(conn: psycopg.Connection, batch_size: int) -> None:
steps = load_steps()
totals = {name: 0 for name, _ in steps}
with conn.cursor() as cur:
cur.execute("SELECT count(*) FROM remote_cam.metadata")
total_rows = cur.fetchone()[0]
cur.execute(
"CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS"
)
conn.commit()
last = ""
seen = 0
t0 = time.perf_counter()
while True:
with conn.transaction(), conn.cursor() as cur:
cur.execute(
"""
INSERT INTO _mig_batch
SELECT metadata_uuid FROM remote_cam.metadata
WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s
""",
(last, batch_size),
)
n = cur.rowcount
if n == 0:
break
cur.execute("SELECT max(metadata_uuid) FROM _mig_batch")
last = cur.fetchone()[0]
added = []
for name, sql in steps:
cur.execute(sql)
totals[name] += cur.rowcount
added.append(f"{name}+{cur.rowcount}")
seen += n
rate = seen / max(time.perf_counter() - t0, 1e-6)
print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True)
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
def verify(conn: psycopg.Connection) -> bool:
checks = [
("metadata == image(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"),
("metadata == camera_status(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
("category == detection(entrance)",
"SELECT (SELECT count(*) FROM remote_cam.category), "
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
("metadata ohne ocr_ts == image FAIL",
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
("keine 0/0-Koordinaten (image, camera_status)",
"SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + "
"(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"),
("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)",
"""
SELECT 0, count(*) FROM (
SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200
) m
JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid
WHERE v.status_reported_at IS DISTINCT FROM m.servertime
OR v.battery_pct IS DISTINCT FROM m."BatteryStatus"
OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion"
OR v.status_camera_name IS DISTINCT FROM m."CameraName"
OR v.signal_strength IS DISTINCT FROM m."SignalStrength"
OR v.temperature_c IS DISTINCT FROM m."Temperature"
OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone"
OR v.ocr_ts IS DISTINCT FROM m.ocr_ts
"""),
]
ok_all = True
with conn.cursor() as cur:
for label, sql in checks:
cur.execute(sql)
expected, actual = cur.fetchone()
ok = expected == actual
ok_all &= ok
print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True)
conn.rollback()
return ok_all
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
ap.add_argument("--batch", type=int, default=20000)
ap.add_argument("--verify-only", action="store_true")
args = ap.parse_args()
if not args.dsn:
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
return 2
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
if not args.verify_only:
apply_schema(conn)
print("Schema angewendet. Migriere ...", flush=True)
migrate(conn, args.batch)
print("Verifikation:", flush=True)
return 0 if verify(conn) else 1
if __name__ == "__main__":
raise SystemExit(main())