Add Gundi v2 push receiver for ICU and normalized data model
ICU now pushes camera events via a Gundi-v2-compatible API instead of dropping file triplets into S3. The new gundi-receiver service accepts them directly, so images are in the system within seconds instead of waiting out the 90s age gate + poll interval. OCR of the burnt-in camera timestamp stays mandatory (ICU only sends server times; OCR detects drifting camera clocks) but now runs asynchronously after ingest. The push path writes a normalized model that separates what the legacy metadata table conflates: camera_status (status time series), image (image properties) and detection (classifications), plus the image_with_status view as the portal's migration target. A repeatable, batched migration splits existing metadata/category rows into it. The legacy poller is unchanged and keeps writing metadata/category until the portal is switched over. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Spielt die Beispiel-Payloads aus der ICU-Doku gegen den Gundi-Receiver.
|
||||
|
||||
python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg bild.jpg
|
||||
|
||||
Ohne --jpeg wird ein synthetisches Bild mit Zeitstempel-Overlay erzeugt (braucht Pillow).
|
||||
Standard-IMEI ist eine Test-IMEI, damit echte Kamerazeitreihen unberuehrt bleiben.
|
||||
"""
|
||||
import argparse
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import urlparse
|
||||
|
||||
TEST_IMEI = "999000000000001"
|
||||
|
||||
|
||||
def request(url: str, method: str, path: str, *, body: bytes = b"", headers: dict | None = None):
|
||||
u = urlparse(url)
|
||||
cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
|
||||
conn = cls(u.hostname, u.port, timeout=30)
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=headers or {})
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
return resp.status, dict(resp.getheaders()), data
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def post_json(url, path, key, payload):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key is not None:
|
||||
headers["apikey"] = key
|
||||
return request(url, "POST", path, body=json.dumps(payload).encode(), headers=headers)
|
||||
|
||||
|
||||
def post_file(url, path, key, jpg: bytes):
|
||||
boundary = f"----smoke-{uuid.uuid4().hex}"
|
||||
body = (
|
||||
f"--{boundary}\r\n"
|
||||
f'Content-Disposition: form-data; name="file1"; filename="../../evil.jpg"\r\n'
|
||||
f"Content-Type: image/jpeg\r\n\r\n"
|
||||
).encode() + jpg + f"\r\n--{boundary}--\r\n".encode()
|
||||
return request(url, "POST", path, body=body, headers={
|
||||
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
||||
"apikey": key,
|
||||
})
|
||||
|
||||
|
||||
def synthetic_jpeg(ts: datetime) -> bytes:
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
w, h = 1920, 1080
|
||||
im = Image.new("RGB", (w, h), (70, 90, 70))
|
||||
draw = ImageDraw.Draw(im)
|
||||
bar = int(h * 0.05)
|
||||
draw.rectangle((0, h - bar, w, h), fill=(0, 0, 0))
|
||||
font = ImageFont.load_default(size=int(bar * 0.8))
|
||||
text = ts.strftime("%H:%M:%S %d-%m-%Y")
|
||||
tw = draw.textlength(text, font=font)
|
||||
draw.text((w - tw - 20, h - bar + bar * 0.08), text, fill=(255, 255, 255), font=font)
|
||||
out = io.BytesIO()
|
||||
im.save(out, format="JPEG", quality=90)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default="http://127.0.0.1:8090")
|
||||
ap.add_argument("--key", default=os.environ.get("GUNDI_API_KEY"))
|
||||
ap.add_argument("--imei", default=TEST_IMEI)
|
||||
ap.add_argument("--jpeg", help="Pfad zu einem echten Kamerabild")
|
||||
args = ap.parse_args()
|
||||
if not args.key:
|
||||
print("--key oder GUNDI_API_KEY fehlt", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
failures = 0
|
||||
|
||||
def check(label: str, ok: bool, detail: str = "") -> None:
|
||||
nonlocal failures
|
||||
failures += 0 if ok else 1
|
||||
print(f"[{'OK ' if ok else 'FAIL'}] {label}{(' - ' + detail) if detail else ''}")
|
||||
|
||||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
status_payload = {
|
||||
"source": args.imei,
|
||||
"event_type": "camera_status_report",
|
||||
"title": "Camera Status report - Smoke Test",
|
||||
"recorded_at": (now - timedelta(minutes=10)).isoformat().replace("+00:00", "Z"),
|
||||
"location": {"lat": 0.0, "lon": 0.0},
|
||||
"event_details": {
|
||||
"imei": args.imei, "iccid": "8900000000000000000", "init_key": "000000000000",
|
||||
"camera_name": "Smoke Test", "camera_model": None, "firmware_version": "1.4.2",
|
||||
"mobile_country_code": "232", "mobile_network_code": "01", "location_area_code": "1234",
|
||||
"cell_id": "567890", "customer_timezone": "Europe/Vienna", "battery_status": 87,
|
||||
"signal_strength": 3, "temperature": -4, "coordinate_switch": False, "work_period": True,
|
||||
"image_format": "F4_3", "image_night": "MIN", "work_start": "06:00", "work_end": "21:00",
|
||||
"thumbnail_size": "720p", "pir_interval": "30", "time_scan": "120", "camera_language": "en",
|
||||
},
|
||||
}
|
||||
trap_payload = {
|
||||
"source": args.imei,
|
||||
"event_type": "cameratrap_rep",
|
||||
"title": "Camera trap image - Smoke Test",
|
||||
"recorded_at": now.isoformat().replace("+00:00", "Z"),
|
||||
"location": {"lat": 47.8095, "lon": 13.0550},
|
||||
"event_details": {
|
||||
"cameratraprep_camera-name": "Smoke Test",
|
||||
"cameratraprep_camera-make": "M350GW",
|
||||
"detections": [
|
||||
{"category": "deer", "confidence_percent": 96.2, "x_px": 0, "y_px": 0, "width_px": 0, "height_px": 0},
|
||||
{"category": "human", "confidence_percent": 3.1, "x_px": 15, "y_px": 17, "width_px": 100, "height_px": 200},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
code, _, _ = request(args.url, "GET", "/healthz")
|
||||
check("healthz", code == 200, f"HTTP {code}")
|
||||
|
||||
code, _, _ = post_json(args.url, "/v2/events/", None, status_payload)
|
||||
check("ohne apikey -> 401", code == 401, f"HTTP {code}")
|
||||
code, _, _ = post_json(args.url, "/v2/events/", "falsch", status_payload)
|
||||
check("falscher apikey -> 401", code == 401, f"HTTP {code}")
|
||||
|
||||
code, headers, _ = post_json(args.url, "/v2/events", args.key, status_payload)
|
||||
check("ohne Slash -> 308", code == 308 and headers.get("Location") == "/v2/events/",
|
||||
f"HTTP {code} Location={headers.get('Location')}")
|
||||
|
||||
code, _, data = post_json(args.url, "/v2/events/", args.key, {"event_type": "cameratrap_rep"})
|
||||
check("unvollstaendiges Event -> 400", code == 400, f"HTTP {code} {data[:120]!r}")
|
||||
|
||||
code, _, data = post_json(args.url, "/v2/events/", args.key, status_payload)
|
||||
status_id = json.loads(data).get("object_id") if code == 200 else None
|
||||
check("camera_status_report", code == 200 and bool(status_id), f"HTTP {code} object_id={status_id}")
|
||||
code, _, _ = post_json(args.url, "/v2/events/", args.key, status_payload)
|
||||
check("camera_status_report erneut (idempotent in camera_status)", code == 200, f"HTTP {code}")
|
||||
|
||||
code, _, data = post_json(args.url, "/v2/events/", args.key, trap_payload)
|
||||
object_id = json.loads(data).get("object_id") if code == 200 else None
|
||||
check("cameratrap_rep", code == 200 and bool(object_id), f"HTTP {code} object_id={object_id}")
|
||||
|
||||
if args.jpeg:
|
||||
with open(args.jpeg, "rb") as f:
|
||||
jpg = f.read()
|
||||
else:
|
||||
jpg = synthetic_jpeg(now.astimezone(timezone(timedelta(hours=2))))
|
||||
|
||||
code, _, _ = post_file(args.url, f"/v2/events/{uuid.uuid4()}/attachments/", args.key, jpg)
|
||||
check("Attachment auf unbekannte object_id -> 404", code == 404, f"HTTP {code}")
|
||||
code, _, _ = post_file(args.url, "/v2/events/..%2F..%2Fx/attachments/", args.key, jpg)
|
||||
check("Attachment auf ungueltige object_id -> 404", code == 404, f"HTTP {code}")
|
||||
code, _, _ = post_file(args.url, f"/v2/events/{status_id}/attachments/", args.key, jpg)
|
||||
check("Attachment auf Status-Event -> 404", code == 404, f"HTTP {code}")
|
||||
|
||||
if object_id:
|
||||
code, headers, _ = post_file(args.url, f"/v2/events/{object_id}/attachments", args.key, jpg)
|
||||
check("Attachment ohne Slash -> 308", code == 308, f"HTTP {code}")
|
||||
code, _, data = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg)
|
||||
check("Attachment", code == 200, f"HTTP {code} {data[:120]!r}")
|
||||
code, _, _ = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg)
|
||||
check("Attachment erneut (duplicate, 200)", code == 200, f"HTTP {code}")
|
||||
|
||||
print(f"\nobject_id cameratrap_rep: {object_id}")
|
||||
print(f"object_id status_report: {status_id}")
|
||||
print(f"{failures} Fehler")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection).
|
||||
|
||||
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
|
||||
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
|
||||
python -m tools.migrate_legacy --verify-only
|
||||
|
||||
Wiederholbar. Jeder Batch ist eine eigene Transaktion.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
|
||||
SCHEMA_LOCK_KEY = 7_311_000_001
|
||||
|
||||
|
||||
def load_steps() -> list[tuple[str, str]]:
|
||||
sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8")
|
||||
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
|
||||
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
|
||||
|
||||
|
||||
def apply_schema(conn: psycopg.Connection) -> None:
|
||||
sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8")
|
||||
with conn.transaction(), conn.cursor() as cur:
|
||||
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
def migrate(conn: psycopg.Connection, batch_size: int) -> None:
|
||||
steps = load_steps()
|
||||
totals = {name: 0 for name, _ in steps}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT count(*) FROM remote_cam.metadata")
|
||||
total_rows = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
last = ""
|
||||
seen = 0
|
||||
t0 = time.perf_counter()
|
||||
while True:
|
||||
with conn.transaction(), conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO _mig_batch
|
||||
SELECT metadata_uuid FROM remote_cam.metadata
|
||||
WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s
|
||||
""",
|
||||
(last, batch_size),
|
||||
)
|
||||
n = cur.rowcount
|
||||
if n == 0:
|
||||
break
|
||||
cur.execute("SELECT max(metadata_uuid) FROM _mig_batch")
|
||||
last = cur.fetchone()[0]
|
||||
added = []
|
||||
for name, sql in steps:
|
||||
cur.execute(sql)
|
||||
totals[name] += cur.rowcount
|
||||
added.append(f"{name}+{cur.rowcount}")
|
||||
seen += n
|
||||
rate = seen / max(time.perf_counter() - t0, 1e-6)
|
||||
print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True)
|
||||
|
||||
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
|
||||
|
||||
|
||||
def verify(conn: psycopg.Connection) -> bool:
|
||||
checks = [
|
||||
("metadata == image(entrance)",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
||||
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"),
|
||||
("metadata == camera_status(entrance)",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
||||
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
|
||||
("category == detection(entrance)",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.category), "
|
||||
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
|
||||
("metadata ohne ocr_ts == image FAIL",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
|
||||
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
|
||||
("keine 0/0-Koordinaten (image, camera_status)",
|
||||
"SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + "
|
||||
"(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"),
|
||||
("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)",
|
||||
"""
|
||||
SELECT 0, count(*) FROM (
|
||||
SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200
|
||||
) m
|
||||
JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid
|
||||
WHERE v.status_reported_at IS DISTINCT FROM m.servertime
|
||||
OR v.battery_pct IS DISTINCT FROM m."BatteryStatus"
|
||||
OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion"
|
||||
OR v.status_camera_name IS DISTINCT FROM m."CameraName"
|
||||
OR v.signal_strength IS DISTINCT FROM m."SignalStrength"
|
||||
OR v.temperature_c IS DISTINCT FROM m."Temperature"
|
||||
OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone"
|
||||
OR v.ocr_ts IS DISTINCT FROM m.ocr_ts
|
||||
"""),
|
||||
]
|
||||
ok_all = True
|
||||
with conn.cursor() as cur:
|
||||
for label, sql in checks:
|
||||
cur.execute(sql)
|
||||
expected, actual = cur.fetchone()
|
||||
ok = expected == actual
|
||||
ok_all &= ok
|
||||
print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True)
|
||||
conn.rollback()
|
||||
return ok_all
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
|
||||
ap.add_argument("--batch", type=int, default=20000)
|
||||
ap.add_argument("--verify-only", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if not args.dsn:
|
||||
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
|
||||
if not args.verify_only:
|
||||
apply_schema(conn)
|
||||
print("Schema angewendet. Migriere ...", flush=True)
|
||||
migrate(conn, args.batch)
|
||||
print("Verifikation:", flush=True)
|
||||
return 0 if verify(conn) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user