Add Gundi v2 push receiver for ICU and normalized data model
ICU now pushes camera events via a Gundi-v2-compatible API instead of dropping file triplets into S3. The new gundi-receiver service accepts them directly, so images are in the system within seconds instead of waiting out the 90s age gate + poll interval. OCR of the burnt-in camera timestamp stays mandatory (ICU only sends server times; OCR detects drifting camera clocks) but now runs asynchronously after ingest. The push path writes a normalized model that separates what the legacy metadata table conflates: camera_status (status time series), image (image properties) and detection (classifications), plus the image_with_status view as the portal's migration target. A repeatable, batched migration splits existing metadata/category rows into it. The legacy poller is unchanged and keeps writing metadata/category until the portal is switched over. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gundi-v2-kompatibler Empfaenger fuer ICU-Push-Events.
|
||||
|
||||
POST /v2/events/ -> {"object_id", "created_at"}
|
||||
POST /v2/events/<object_id>/attachments/ -> {"object_id", "created_at"}
|
||||
GET /healthz
|
||||
"""
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from email.parser import BytesParser
|
||||
from email.policy import default as email_policy
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from gundi_pipeline import (
|
||||
UUID_STRICT_RE,
|
||||
EventError,
|
||||
GundiConfig,
|
||||
Pipeline,
|
||||
apply_schema,
|
||||
attach_image,
|
||||
load_gundi_config,
|
||||
store_events,
|
||||
utc_now_str,
|
||||
validate_event,
|
||||
)
|
||||
from main import AppConfig, load_config
|
||||
|
||||
EVENTS_PATH = "/v2/events/"
|
||||
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
|
||||
def extract_jpeg(content_type: str, body: bytes) -> tuple[bytes | None, int]:
|
||||
"""Erstes JPEG aus einem multipart/form-data-Body; Client-Dateinamen werden ignoriert."""
|
||||
raw = f"Content-Type: {content_type}\r\nMIME-Version: 1.0\r\n\r\n".encode("latin-1") + body
|
||||
msg = BytesParser(policy=email_policy).parsebytes(raw)
|
||||
if not msg.is_multipart():
|
||||
return None, 0
|
||||
files = []
|
||||
for part in msg.iter_parts():
|
||||
if part.get_content_disposition() != "form-data" or not part.get_filename():
|
||||
continue
|
||||
files.append(part.get_payload(decode=True) or b"")
|
||||
jpgs = [f for f in files if f.startswith(JPEG_MAGIC)]
|
||||
return (jpgs[0] if jpgs else None), len(files)
|
||||
|
||||
|
||||
class GundiServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, addr, cfg: AppConfig, gcfg: GundiConfig, pipeline: Pipeline):
|
||||
super().__init__(addr, GundiHandler)
|
||||
self.cfg = cfg
|
||||
self.gcfg = gcfg
|
||||
self.pipeline = pipeline
|
||||
|
||||
|
||||
class GundiHandler(BaseHTTPRequestHandler):
|
||||
server: GundiServer
|
||||
server_version = "melesICUmover-gundi/1.0"
|
||||
sys_version = ""
|
||||
timeout = 60
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
client = self.headers.get("X-Real-IP") if self.headers else None
|
||||
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
|
||||
|
||||
# --- Antworten -------------------------------------------------------
|
||||
def _json(self, code: int, data) -> None:
|
||||
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _error(self, code: int, message: str, *, close: bool = False) -> None:
|
||||
if close:
|
||||
self.close_connection = True
|
||||
self._json(code, {"error": message})
|
||||
|
||||
def _redirect(self, location: str) -> None:
|
||||
# 308 statt 301: der Client muss POST + Body beibehalten.
|
||||
self.close_connection = True
|
||||
self.send_response(308)
|
||||
self.send_header("Location", location)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# --- Request-Helfer --------------------------------------------------
|
||||
def _authorized(self) -> bool:
|
||||
got = (self.headers.get("apikey") or "").encode("utf-8")
|
||||
return hmac.compare_digest(got, self.server.gcfg.api_key.encode("utf-8"))
|
||||
|
||||
def _read_body(self) -> bytes | None:
|
||||
if self.headers.get("Transfer-Encoding"):
|
||||
self._error(411, "Transfer-Encoding wird nicht unterstuetzt, Content-Length senden", close=True)
|
||||
return None
|
||||
raw_len = self.headers.get("Content-Length")
|
||||
if raw_len is None:
|
||||
self._error(411, "Content-Length erforderlich", close=True)
|
||||
return None
|
||||
try:
|
||||
length = int(raw_len)
|
||||
except ValueError:
|
||||
self._error(400, "Content-Length ungueltig", close=True)
|
||||
return None
|
||||
if length < 0:
|
||||
self._error(400, "Content-Length ungueltig", close=True)
|
||||
return None
|
||||
if length > self.server.gcfg.max_request_bytes:
|
||||
self._error(413, "Request zu gross", close=True)
|
||||
return None
|
||||
body = self.rfile.read(length)
|
||||
if len(body) != length:
|
||||
self._error(400, "Body unvollstaendig", close=True)
|
||||
return None
|
||||
return body
|
||||
|
||||
# --- Routing ---------------------------------------------------------
|
||||
def do_GET(self) -> None:
|
||||
if urlparse(self.path).path == "/healthz":
|
||||
self._json(200, {"status": "ok"})
|
||||
else:
|
||||
self._error(404, "not found")
|
||||
|
||||
def do_POST(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if path == "/v2/events":
|
||||
self._redirect(EVENTS_PATH)
|
||||
return
|
||||
attachment = ATTACHMENT_RE.match(path)
|
||||
if attachment and not attachment.group(2):
|
||||
self._redirect(path + "/")
|
||||
return
|
||||
if path != EVENTS_PATH and not attachment:
|
||||
self._error(404, "not found", close=True)
|
||||
return
|
||||
if not self._authorized():
|
||||
self._error(401, "Invalid or missing apikey", close=True)
|
||||
return
|
||||
body = self._read_body()
|
||||
if body is None:
|
||||
return
|
||||
try:
|
||||
if attachment:
|
||||
self._handle_attachment(attachment.group(1).lower(), body)
|
||||
else:
|
||||
self._handle_events(body)
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(500, "internal error", close=True)
|
||||
|
||||
# --- Handler ---------------------------------------------------------
|
||||
def _handle_events(self, body: bytes) -> None:
|
||||
if "application/json" not in (self.headers.get("Content-Type") or "").lower():
|
||||
self._error(415, "Content-Type muss application/json sein")
|
||||
return
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
self._error(400, "Ungueltiges JSON")
|
||||
return
|
||||
events = payload if isinstance(payload, list) else [payload]
|
||||
if not events:
|
||||
self._error(400, "Leere Event-Liste")
|
||||
return
|
||||
try:
|
||||
envs = [validate_event(ev) for ev in events]
|
||||
except EventError as e:
|
||||
self._error(400, str(e))
|
||||
return
|
||||
|
||||
results = store_events(self.server.cfg.pg_dsn, list(zip(envs, events)))
|
||||
for r in results:
|
||||
print(
|
||||
f"[GUNDI] event object_id={r['object_id']} type={r['event_type']} "
|
||||
f"imei={r['source']} state={r['state']}",
|
||||
flush=True,
|
||||
)
|
||||
out = [{"object_id": r["object_id"], "created_at": r["created_at"]} for r in results]
|
||||
self._json(200, out if isinstance(payload, list) else out[0])
|
||||
|
||||
def _handle_attachment(self, object_id: str, body: bytes) -> None:
|
||||
if not UUID_STRICT_RE.match(object_id):
|
||||
self._error(404, "Unknown object_id")
|
||||
return
|
||||
content_type = self.headers.get("Content-Type") or ""
|
||||
if "multipart/form-data" not in content_type.lower():
|
||||
self._error(415, "Content-Type muss multipart/form-data sein")
|
||||
return
|
||||
jpg, n_files = extract_jpeg(content_type, body)
|
||||
if jpg is None:
|
||||
self._error(400, "Kein JPEG-Anhang gefunden")
|
||||
return
|
||||
try:
|
||||
outcome = attach_image(
|
||||
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg
|
||||
)
|
||||
except LookupError:
|
||||
self._error(404, "Unknown object_id")
|
||||
return
|
||||
if outcome == "stored":
|
||||
self.server.pipeline.submit_ingest(object_id)
|
||||
print(
|
||||
f"[GUNDI] attachment object_id={object_id} bytes={len(jpg)} files={n_files} outcome={outcome}",
|
||||
flush=True,
|
||||
)
|
||||
self._json(200, {"object_id": object_id, "created_at": utc_now_str()})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
|
||||
cfg = load_config(cfg_path)
|
||||
gcfg = load_gundi_config(cfg_path)
|
||||
if not gcfg.enabled:
|
||||
print("[GUNDI] gundi.enabled ist false - Receiver bleibt inaktiv.", flush=True)
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
|
||||
apply_schema(cfg.pg_dsn)
|
||||
pipeline = Pipeline(cfg, gcfg)
|
||||
pipeline.start()
|
||||
server = GundiServer((gcfg.bind_host, gcfg.bind_port), cfg, gcfg, pipeline)
|
||||
|
||||
def _shutdown(signum, frame):
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, _shutdown)
|
||||
signal.signal(signal.SIGINT, _shutdown)
|
||||
|
||||
print(
|
||||
f"[GUNDI] lausche auf {gcfg.bind_host}:{gcfg.bind_port} "
|
||||
f"inbox={gcfg.inbox_prefix} processed={gcfg.processed_prefix} thumbs={gcfg.thumb_prefix}",
|
||||
flush=True,
|
||||
)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
pipeline.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user