Add Gundi v2 push receiver for ICU and normalized data model

ICU now pushes camera events via a Gundi-v2-compatible API instead of
dropping file triplets into S3. The new gundi-receiver service accepts
them directly, so images are in the system within seconds instead of
waiting out the 90s age gate + poll interval. OCR of the burnt-in camera
timestamp stays mandatory (ICU only sends server times; OCR detects
drifting camera clocks) but now runs asynchronously after ingest.

The push path writes a normalized model that separates what the legacy
metadata table conflates: camera_status (status time series), image
(image properties) and detection (classifications), plus the
image_with_status view as the portal's migration target. A repeatable,
batched migration splits existing metadata/category rows into it.

The legacy poller is unchanged and keeps writing metadata/category until
the portal is switched over.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-09-18 13:54:21 +00:00
co-authored by Claude Opus 5
parent 94bfc60fc4
commit 2fc8dea10c
13 changed files with 1638 additions and 11 deletions
+252
View File
@@ -0,0 +1,252 @@
#!/usr/bin/env python3
"""Gundi-v2-kompatibler Empfaenger fuer ICU-Push-Events.
POST /v2/events/ -> {"object_id", "created_at"}
POST /v2/events/<object_id>/attachments/ -> {"object_id", "created_at"}
GET /healthz
"""
import hmac
import json
import os
import re
import signal
import threading
import time
from email.parser import BytesParser
from email.policy import default as email_policy
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlparse
from gundi_pipeline import (
UUID_STRICT_RE,
EventError,
GundiConfig,
Pipeline,
apply_schema,
attach_image,
load_gundi_config,
store_events,
utc_now_str,
validate_event,
)
from main import AppConfig, load_config
EVENTS_PATH = "/v2/events/"
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
JPEG_MAGIC = b"\xff\xd8\xff"
def extract_jpeg(content_type: str, body: bytes) -> tuple[bytes | None, int]:
"""Erstes JPEG aus einem multipart/form-data-Body; Client-Dateinamen werden ignoriert."""
raw = f"Content-Type: {content_type}\r\nMIME-Version: 1.0\r\n\r\n".encode("latin-1") + body
msg = BytesParser(policy=email_policy).parsebytes(raw)
if not msg.is_multipart():
return None, 0
files = []
for part in msg.iter_parts():
if part.get_content_disposition() != "form-data" or not part.get_filename():
continue
files.append(part.get_payload(decode=True) or b"")
jpgs = [f for f in files if f.startswith(JPEG_MAGIC)]
return (jpgs[0] if jpgs else None), len(files)
class GundiServer(ThreadingHTTPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(self, addr, cfg: AppConfig, gcfg: GundiConfig, pipeline: Pipeline):
super().__init__(addr, GundiHandler)
self.cfg = cfg
self.gcfg = gcfg
self.pipeline = pipeline
class GundiHandler(BaseHTTPRequestHandler):
server: GundiServer
server_version = "melesICUmover-gundi/1.0"
sys_version = ""
timeout = 60
def log_message(self, fmt, *args):
client = self.headers.get("X-Real-IP") if self.headers else None
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
# --- Antworten -------------------------------------------------------
def _json(self, code: int, data) -> None:
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _error(self, code: int, message: str, *, close: bool = False) -> None:
if close:
self.close_connection = True
self._json(code, {"error": message})
def _redirect(self, location: str) -> None:
# 308 statt 301: der Client muss POST + Body beibehalten.
self.close_connection = True
self.send_response(308)
self.send_header("Location", location)
self.send_header("Content-Length", "0")
self.end_headers()
# --- Request-Helfer --------------------------------------------------
def _authorized(self) -> bool:
got = (self.headers.get("apikey") or "").encode("utf-8")
return hmac.compare_digest(got, self.server.gcfg.api_key.encode("utf-8"))
def _read_body(self) -> bytes | None:
if self.headers.get("Transfer-Encoding"):
self._error(411, "Transfer-Encoding wird nicht unterstuetzt, Content-Length senden", close=True)
return None
raw_len = self.headers.get("Content-Length")
if raw_len is None:
self._error(411, "Content-Length erforderlich", close=True)
return None
try:
length = int(raw_len)
except ValueError:
self._error(400, "Content-Length ungueltig", close=True)
return None
if length < 0:
self._error(400, "Content-Length ungueltig", close=True)
return None
if length > self.server.gcfg.max_request_bytes:
self._error(413, "Request zu gross", close=True)
return None
body = self.rfile.read(length)
if len(body) != length:
self._error(400, "Body unvollstaendig", close=True)
return None
return body
# --- Routing ---------------------------------------------------------
def do_GET(self) -> None:
if urlparse(self.path).path == "/healthz":
self._json(200, {"status": "ok"})
else:
self._error(404, "not found")
def do_POST(self) -> None:
path = urlparse(self.path).path
if path == "/v2/events":
self._redirect(EVENTS_PATH)
return
attachment = ATTACHMENT_RE.match(path)
if attachment and not attachment.group(2):
self._redirect(path + "/")
return
if path != EVENTS_PATH and not attachment:
self._error(404, "not found", close=True)
return
if not self._authorized():
self._error(401, "Invalid or missing apikey", close=True)
return
body = self._read_body()
if body is None:
return
try:
if attachment:
self._handle_attachment(attachment.group(1).lower(), body)
else:
self._handle_events(body)
except Exception as e:
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
self._error(500, "internal error", close=True)
# --- Handler ---------------------------------------------------------
def _handle_events(self, body: bytes) -> None:
if "application/json" not in (self.headers.get("Content-Type") or "").lower():
self._error(415, "Content-Type muss application/json sein")
return
try:
payload = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
self._error(400, "Ungueltiges JSON")
return
events = payload if isinstance(payload, list) else [payload]
if not events:
self._error(400, "Leere Event-Liste")
return
try:
envs = [validate_event(ev) for ev in events]
except EventError as e:
self._error(400, str(e))
return
results = store_events(self.server.cfg.pg_dsn, list(zip(envs, events)))
for r in results:
print(
f"[GUNDI] event object_id={r['object_id']} type={r['event_type']} "
f"imei={r['source']} state={r['state']}",
flush=True,
)
out = [{"object_id": r["object_id"], "created_at": r["created_at"]} for r in results]
self._json(200, out if isinstance(payload, list) else out[0])
def _handle_attachment(self, object_id: str, body: bytes) -> None:
if not UUID_STRICT_RE.match(object_id):
self._error(404, "Unknown object_id")
return
content_type = self.headers.get("Content-Type") or ""
if "multipart/form-data" not in content_type.lower():
self._error(415, "Content-Type muss multipart/form-data sein")
return
jpg, n_files = extract_jpeg(content_type, body)
if jpg is None:
self._error(400, "Kein JPEG-Anhang gefunden")
return
try:
outcome = attach_image(
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg
)
except LookupError:
self._error(404, "Unknown object_id")
return
if outcome == "stored":
self.server.pipeline.submit_ingest(object_id)
print(
f"[GUNDI] attachment object_id={object_id} bytes={len(jpg)} files={n_files} outcome={outcome}",
flush=True,
)
self._json(200, {"object_id": object_id, "created_at": utc_now_str()})
def main() -> None:
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
cfg = load_config(cfg_path)
gcfg = load_gundi_config(cfg_path)
if not gcfg.enabled:
print("[GUNDI] gundi.enabled ist false - Receiver bleibt inaktiv.", flush=True)
while True:
time.sleep(3600)
apply_schema(cfg.pg_dsn)
pipeline = Pipeline(cfg, gcfg)
pipeline.start()
server = GundiServer((gcfg.bind_host, gcfg.bind_port), cfg, gcfg, pipeline)
def _shutdown(signum, frame):
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, _shutdown)
signal.signal(signal.SIGINT, _shutdown)
print(
f"[GUNDI] lausche auf {gcfg.bind_host}:{gcfg.bind_port} "
f"inbox={gcfg.inbox_prefix} processed={gcfg.processed_prefix} thumbs={gcfg.thumb_prefix}",
flush=True,
)
try:
server.serve_forever()
finally:
server.server_close()
pipeline.stop()
if __name__ == "__main__":
main()