"""Push-Pfad (Gundi v2): Mapping ins Fachmodell, Ingest (Stufe 1), OCR (Stufe 2), Sweep.""" import os import re import subprocess import tempfile import threading import time import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Optional import psycopg import yaml from dateutil import tz from psycopg.types.json import Json from main import ( AppConfig, S3, exif_write_with_exiftool, make_thumbnail_bytes, ocr_extract_timestamp, to_exif_datetime, ) SCHEMA_FILE = Path(__file__).resolve().parent / "schema" / "001_domain_model.sql" SCHEMA_LOCK_KEY = 7_311_000_001 UUID_STRICT_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") EVENT_CAMERATRAP = "cameratrap_rep" EVENT_STATUS = "camera_status_report" MIN_API_KEY_LEN = 24 # ----------------------- # Config # ----------------------- @dataclass class GundiConfig: enabled: bool api_key: str bind_host: str = "0.0.0.0" bind_port: int = 8090 inbox_prefix: str = "icu/gundi-test/inbox/" processed_prefix: str = "icu/gundi-test/processed/" thumb_prefix: str = "icu/gundi-test/thumbnails/" workers: int = 4 ocr_workers: int = 2 max_request_bytes: int = 32 * 1024 * 1024 abandon_after_hours: int = 24 sweep_seconds: int = 60 stale_after_seconds: int = 300 max_attempts: int = 3 def load_gundi_config(path: str) -> GundiConfig: with open(path, "r", encoding="utf-8") as f: raw = yaml.safe_load(f) or {} g = raw.get("gundi") or {} defaults = GundiConfig(enabled=False, api_key="") def prefix(key: str) -> str: value = str(g.get(key, getattr(defaults, key))) return value if value.endswith("/") else value + "/" gcfg = GundiConfig( enabled=bool(g.get("enabled", False)), api_key=os.environ.get("GUNDI_API_KEY", ""), bind_host=str(g.get("bind_host", defaults.bind_host)), bind_port=int(g.get("bind_port", defaults.bind_port)), inbox_prefix=prefix("inbox_prefix"), processed_prefix=prefix("processed_prefix"), thumb_prefix=prefix("thumb_prefix"), workers=max(1, int(g.get("workers", defaults.workers))), ocr_workers=max(1, int(g.get("ocr_workers", defaults.ocr_workers))), max_request_bytes=int(g.get("max_request_bytes", defaults.max_request_bytes)), abandon_after_hours=int(g.get("abandon_after_hours", defaults.abandon_after_hours)), sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)), stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)), max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))), ) if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN: raise SystemExit( f"GUNDI_API_KEY fehlt oder ist kuerzer als {MIN_API_KEY_LEN} Zeichen " "(gundi.enabled ist aktiv). Bitte in der .env setzen." ) return gcfg def connect(dsn: str) -> psycopg.Connection: return psycopg.connect(dsn, connect_timeout=10) def apply_schema(dsn: str) -> None: sql = SCHEMA_FILE.read_text(encoding="utf-8") with connect(dsn) as conn, conn.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,)) cur.execute(sql) def utc_now_str() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") # ----------------------- # Validierung + Mapping # ----------------------- class EventError(ValueError): pass def _num(v) -> Optional[float]: if isinstance(v, bool) or not isinstance(v, (int, float)): return None return float(v) def _int(v) -> Optional[int]: if isinstance(v, bool): return None if isinstance(v, int): return v if isinstance(v, float) and v.is_integer(): return int(v) if isinstance(v, str) and re.fullmatch(r"-?\d+", v.strip()): return int(v.strip()) return None def _bool(v) -> Optional[bool]: if isinstance(v, bool): return v if isinstance(v, str) and v.strip().lower() in ("true", "false"): return v.strip().lower() == "true" return None def _str(v) -> Optional[str]: if v is None: return None s = str(v).strip() return s or None def parse_ts(value) -> datetime: if not isinstance(value, str) or not value.strip(): raise EventError("recorded_at fehlt") s = value.strip() if s.endswith("Z"): s = s[:-1] + "+00:00" try: dt = datetime.fromisoformat(s) except ValueError: raise EventError(f"recorded_at ungueltig: {value!r}") from None if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) return dt.astimezone(timezone.utc) def coords(location) -> tuple[Optional[float], Optional[float]]: """ICU meldet unbekannte Koordinaten als 0.0/0.0 -> NULL.""" if not isinstance(location, dict): return None, None lat, lon = _num(location.get("lat")), _num(location.get("lon")) if lat is None or lon is None or (lat == 0 and lon == 0): return None, None return lat, lon def validate_event(ev) -> dict: if not isinstance(ev, dict): raise EventError("Event muss ein JSON-Objekt sein") event_type = ev.get("event_type") if not isinstance(event_type, str) or not event_type: raise EventError("event_type fehlt") source = ev.get("source") if not isinstance(source, str) or not source.strip(): raise EventError("source fehlt") details = ev.get("event_details") if details is None: details = {} if not isinstance(details, dict): raise EventError("event_details muss ein Objekt sein") if event_type == EVENT_CAMERATRAP: dets = details.get("detections", []) if not isinstance(dets, list) or not all(isinstance(d, dict) for d in dets): raise EventError("event_details.detections muss eine Liste von Objekten sein") lat, lon = coords(ev.get("location")) return { "event_type": event_type, "source": source.strip(), "recorded_at": parse_ts(ev.get("recorded_at")), "lat": lat, "lon": lon, "details": details, } def camera_status_row(env: dict, object_id: str, raw: dict) -> dict: d = env["details"] return { "imei": env["source"], "reported_at": env["recorded_at"], "iccid": _str(d.get("iccid")), "init_key": _str(d.get("init_key")), "camera_name": _str(d.get("camera_name")), "camera_model": _str(d.get("camera_model")), "firmware_version": _str(d.get("firmware_version")), "camera_language": _str(d.get("camera_language")), "customer_timezone": _str(d.get("customer_timezone")), "mcc": _str(d.get("mobile_country_code")), "mnc": _str(d.get("mobile_network_code")), "lac": _str(d.get("location_area_code")), "cell_id": _str(d.get("cell_id")), "battery_pct": _int(d.get("battery_status")), "signal_strength": _int(d.get("signal_strength")), "temperature_c": _int(d.get("temperature")), "coordinate_switch": _bool(d.get("coordinate_switch")), "work_period": _bool(d.get("work_period")), "work_start": _str(d.get("work_start")), "work_end": _str(d.get("work_end")), "image_format": _str(d.get("image_format")), "image_night": _str(d.get("image_night")), "thumbnail_size": _str(d.get("thumbnail_size")), "pir_interval": _str(d.get("pir_interval")), "time_scan": _str(d.get("time_scan")), "lat": env["lat"], "lon": env["lon"], "source": "gundi", "source_ref": object_id, "raw": Json(raw), } def detection_rows(env: dict, image_uuid: str) -> list[dict]: rows = [] for det in env["details"].get("detections") or []: category = _str(det.get("category")) if category is None: continue pct = _num(det.get("confidence_percent")) confidence = None if pct is None else min(max(pct / 100.0, 0.0), 1.0) bbox = [_int(det.get(k)) for k in ("x_px", "y_px", "width_px", "height_px")] if all(v in (None, 0) for v in bbox): bbox = [None, None, None, None] rows.append({ "image_uuid": image_uuid, "category": category, "confidence": confidence, "bbox_x": bbox[0], "bbox_y": bbox[1], "bbox_w": bbox[2], "bbox_h": bbox[3], "model_version": "icu", "source": "gundi", }) return rows def image_row(env: dict, object_id: str, raw: dict, size_bytes: int, processed_key: str, thumb_key: str) -> dict: d = env["details"] return { "image_uuid": object_id, "imei": env["source"], "servertime": env["recorded_at"], "ocr_state": "PENDING", "camera_name": _str(d.get("cameratraprep_camera-name")), "camera_make": _str(d.get("cameratraprep_camera-make")), "lat": env["lat"], "lon": env["lon"], "size_bytes": size_bytes, "processed_key": processed_key, "thumbnail_key": thumb_key, "source": "gundi", "source_ref": object_id, "raw": Json(raw), } def _insert(cur, table: str, row: dict, suffix: str = "") -> int: cols = list(row) cur.execute( f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) {suffix}", [row[c] for c in cols], ) return cur.rowcount # ----------------------- # Receiver-seitige Persistenz (synchron im Request) # ----------------------- def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]: """Ein Request = eine Transaktion. Antwort erst nach Commit.""" results = [] with connect(dsn) as conn, conn.cursor() as cur: for env, raw in items: object_id = str(uuidlib.uuid4()) if env["event_type"] == EVENT_STATUS: state = "INGESTED" elif env["event_type"] == EVENT_CAMERATRAP: state = "AWAITING_ATTACHMENT" else: state = "IGNORED" _insert(cur, "remote_cam.gundi_inbox", { "object_id": object_id, "event_type": env["event_type"], "source": env["source"], "recorded_at": env["recorded_at"], "payload": Json(raw), "state": state, }) if env["event_type"] == EVENT_STATUS: _insert( cur, "remote_cam.camera_status", camera_status_row(env, object_id, raw), "ON CONFLICT (imei, reported_at) WHERE source = 'gundi' DO NOTHING", ) results.append({ "object_id": object_id, "created_at": utc_now_str(), "event_type": env["event_type"], "source": env["source"], "state": state, }) return results def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str: """Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.""" with connect(dsn) as conn, conn.cursor() as cur: cur.execute( "SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE", (object_id,), ) row = cur.fetchone() if not row or row[0] != EVENT_CAMERATRAP: raise LookupError(object_id) if row[1] in ("RECEIVED", "INGESTED"): return "duplicate" key = f"{gcfg.inbox_prefix}{object_id}.jpg" s3.put_bytes(key, jpg, "image/jpeg") cur.execute( """ UPDATE remote_cam.gundi_inbox SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(), attempts = 0, last_error = NULL, updated_at = now() WHERE object_id = %s """, (key, object_id), ) return "stored" # ----------------------- # EXIF # ----------------------- def exif_set_digitized(jpg: bytes, servertime: datetime, *, timeout_seconds: int) -> bytes: """Stufe 1: nur Serverzeit. DateTimeOriginal folgt nach dem OCR (Stufe 2).""" dtd = to_exif_datetime(servertime.astimezone(timezone.utc)) with tempfile.TemporaryDirectory() as td: path = os.path.join(td, "in.jpg") with open(path, "wb") as f: f.write(jpg) subprocess.run( [ "exiftool", "-overwrite_original", f"-DateTimeDigitized={dtd}", "-OffsetTimeDigitized=+00:00", f"-ModifyDate={dtd}", "-OffsetTime=+00:00", path, ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_seconds, ) with open(path, "rb") as f: return f.read() # ----------------------- # Pipeline # ----------------------- class Pipeline: def __init__(self, cfg: AppConfig, gcfg: GundiConfig): self.cfg = cfg self.gcfg = gcfg self.s3 = S3(cfg) self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest") self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr") self._stop = threading.Event() def start(self) -> None: threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start() def stop(self) -> None: self._stop.set() self._ingest_pool.shutdown(wait=False, cancel_futures=True) self._ocr_pool.shutdown(wait=False, cancel_futures=True) def submit_ingest(self, object_id: str) -> None: self._ingest_pool.submit(self._guarded, self.ingest, object_id) def submit_ocr(self, image_uuid: str) -> None: self._ocr_pool.submit(self._guarded, self.ocr, image_uuid) @staticmethod def _guarded(fn, arg: str) -> None: try: fn(arg) except Exception as e: print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True) # --- Stufe 1 --------------------------------------------------------- def ingest(self, object_id: str) -> None: start = time.perf_counter() try: with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ SELECT payload, attachment_key FROM remote_cam.gundi_inbox WHERE object_id = %s AND state = 'RECEIVED' FOR UPDATE SKIP LOCKED """, (object_id,), ) row = cur.fetchone() if not row: return payload, attachment_key = row env = validate_event(payload) jpg = self.s3.get_bytes(attachment_key) exif_note = "ok" try: processed = exif_set_digitized( jpg, env["recorded_at"], timeout_seconds=self.cfg.exiftool_timeout_seconds ) except Exception as e: processed = jpg exif_note = f"fail({type(e).__name__})" thumb = make_thumbnail_bytes(jpg, self.cfg.thumb_max) processed_key = f"{self.gcfg.processed_prefix}{object_id}.jpg" thumb_key = f"{self.gcfg.thumb_prefix}{object_id}.jpg" self.s3.put_bytes(processed_key, processed, "image/jpeg") self.s3.put_bytes(thumb_key, thumb, "image/jpeg") n_det = 0 inserted = _insert( cur, "remote_cam.image", image_row(env, object_id, payload, len(jpg), processed_key, thumb_key), "ON CONFLICT (image_uuid) DO NOTHING", ) if inserted: for det in detection_rows(env, object_id): n_det += _insert(cur, "remote_cam.detection", det) cur.execute( """ UPDATE remote_cam.gundi_inbox SET state = 'INGESTED', last_error = NULL, updated_at = now() WHERE object_id = %s """, (object_id,), ) except Exception as e: self._record_ingest_failure(object_id, e) return try: self.s3.delete_keys([attachment_key]) except Exception as e: print(f"[GUNDI] inbox-cleanup {object_id} fehlgeschlagen: {e}", flush=True) self.submit_ocr(object_id) ms = int((time.perf_counter() - start) * 1000) print( f"[GUNDI] ingest object_id={object_id} imei={env['source']} detections={n_det} " f"exif={exif_note} ms={ms}", flush=True, ) def _record_ingest_failure(self, object_id: str, err: Exception) -> None: msg = f"{type(err).__name__}: {err}"[:1000] with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ UPDATE remote_cam.gundi_inbox SET attempts = attempts + 1, state = CASE WHEN attempts + 1 >= %s THEN 'FAILED' ELSE state END, last_error = %s, updated_at = now() WHERE object_id = %s AND state = 'RECEIVED' RETURNING attempts, state """, (self.gcfg.max_attempts, msg, object_id), ) row = cur.fetchone() if row: print(f"[GUNDI] ingest object_id={object_id} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True) # --- Stufe 2 --------------------------------------------------------- def ocr(self, image_uuid: str) -> None: start = time.perf_counter() try: with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ SELECT imei, servertime, processed_key FROM remote_cam.image WHERE image_uuid = %s AND ocr_state = 'PENDING' FOR UPDATE SKIP LOCKED """, (image_uuid,), ) row = cur.fetchone() if not row: return imei, servertime, processed_key = row tzinfo, tzname = self._camera_tz(cur, imei, servertime) jpg = self.s3.get_bytes(processed_key) ocr_ts = ocr_extract_timestamp( jpg, self.cfg.ocr_crop_w_frac, self.cfg.ocr_crop_h_frac, tzinfo ) exif_err = None try: patched = exif_write_with_exiftool( jpg, ocr_ts, servertime, timeout_seconds=self.cfg.exiftool_timeout_seconds ) self.s3.put_bytes(processed_key, patched, "image/jpeg") except Exception as e: exif_err = f"exif: {type(e).__name__}: {e}"[:1000] cur.execute( """ UPDATE remote_cam.image SET ocr_ts = %s, ocr_state = 'OK', ocr_attempts = ocr_attempts + 1, ocr_error = %s, updated_at = now() WHERE image_uuid = %s """, (ocr_ts, exif_err, image_uuid), ) except Exception as e: self._record_ocr_failure(image_uuid, e) return drift = int((servertime - ocr_ts).total_seconds()) ms = int((time.perf_counter() - start) * 1000) print( f"[GUNDI] ocr image_uuid={image_uuid} ocr_ts={ocr_ts.isoformat()} tz={tzname} " f"servertime-ocr={drift}s exif={'ok' if exif_err is None else 'fail'} ms={ms}", flush=True, ) @staticmethod def _camera_tz(cur, imei: str, servertime: datetime): cur.execute( """ SELECT customer_timezone FROM remote_cam.camera_status WHERE imei = %s AND customer_timezone IS NOT NULL AND reported_at <= %s ORDER BY reported_at DESC LIMIT 1 """, (imei, servertime), ) row = cur.fetchone() if not row: cur.execute( """ SELECT customer_timezone FROM remote_cam.camera_status WHERE imei = %s AND customer_timezone IS NOT NULL ORDER BY reported_at ASC LIMIT 1 """, (imei,), ) row = cur.fetchone() name = row[0] if row else "UTC" zone = tz.gettz(name) if zone is None: return timezone.utc, "UTC" return zone, name def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None: msg = f"{type(err).__name__}: {err}"[:1000] with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ UPDATE remote_cam.image SET ocr_attempts = ocr_attempts + 1, ocr_state = CASE WHEN ocr_attempts + 1 >= %s THEN 'FAIL' ELSE ocr_state END, ocr_error = %s, updated_at = now() WHERE image_uuid = %s AND ocr_state = 'PENDING' RETURNING ocr_attempts, ocr_state """, (self.gcfg.max_attempts, msg, image_uuid), ) row = cur.fetchone() if row: print(f"[GUNDI] ocr image_uuid={image_uuid} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True) # --- Sweep ----------------------------------------------------------- def _sweep_loop(self) -> None: delay = 0 while not self._stop.wait(delay): delay = self.gcfg.sweep_seconds try: self.sweep() except Exception as e: print(f"[GUNDI] sweep Fehler: {type(e).__name__}: {e}", flush=True) def sweep(self) -> None: stale = self.gcfg.stale_after_seconds with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ SELECT object_id::text FROM remote_cam.gundi_inbox WHERE state = 'RECEIVED' AND updated_at < now() - make_interval(secs => %s) ORDER BY updated_at LIMIT 200 """, (stale,), ) ingest_ids = [r[0] for r in cur.fetchall()] cur.execute( """ SELECT image_uuid::text FROM remote_cam.image WHERE ocr_state = 'PENDING' AND source = 'gundi' AND updated_at < now() - make_interval(secs => %s) ORDER BY updated_at LIMIT 200 """, (stale,), ) ocr_ids = [r[0] for r in cur.fetchall()] cur.execute( """ UPDATE remote_cam.gundi_inbox SET state = 'ABANDONED', updated_at = now() WHERE state = 'AWAITING_ATTACHMENT' AND received_at < now() - make_interval(hours => %s) """, (self.gcfg.abandon_after_hours,), ) abandoned = cur.rowcount for object_id in ingest_ids: self.submit_ingest(object_id) for image_uuid in ocr_ids: self.submit_ocr(image_uuid) if ingest_ids or ocr_ids or abandoned: print( f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}", flush=True, )