diff --git a/.env.example b/.env.example index 5909cd4..6f8b5ea 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,8 @@ PG_DSN=postgresql://USER:PASSWORT@HOST:PORT/meles?sslmode=require # --- DeerMapper API (nur noetig, wenn app.enable_deermapper_api: true) --- DEERMAPPER_API_URL=https://webapp.deermapper.net/api/icu DEERMAPPER_API_KEY=__NEUEN_API_KEY_EINTRAGEN__ + +# --- Gundi-Receiver (Push-Schnittstelle fuer ICU) --- +# Diesen Wert bekommt ICU als "apikey". Mind. 24 Zeichen, z.B.: +# python3 -c "import secrets; print(secrets.token_urlsafe(32))" +GUNDI_API_KEY=__NEUEN_API_KEY_EINTRAGEN__ diff --git a/Dockerfile b/Dockerfile index fbac935..ef91a1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,9 @@ COPY requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt # Code + nicht-geheime Konfiguration ins Image (Secrets kommen zur Laufzeit via ENV). -COPY main.py /app/main.py +COPY main.py gundi_pipeline.py gundi_receiver.py /app/ +COPY schema/ /app/schema/ +COPY tools/ /app/tools/ COPY config.yaml /app/config.yaml # Haertung: nicht als root laufen. diff --git a/README.md b/README.md index 170449f..2e35303 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ Secrets werden **ausschliesslich** ueber Umgebungsvariablen geladen; die Werte i Vollstaendigkeit und bricht mit klarer Meldung ab, falls ein Secret fehlt. Unterstuetzte Umgebungsvariablen: `S3_ENDPOINT`, `S3_BUCKET`, `S3_ACCESS_KEY`, -`S3_SECRET_KEY`, `PG_DSN`, `DEERMAPPER_API_URL`, `DEERMAPPER_API_KEY`, `CONFIG_YAML`, -`DEBUG`. +`S3_SECRET_KEY`, `PG_DSN`, `DEERMAPPER_API_URL`, `DEERMAPPER_API_KEY`, `GUNDI_API_KEY`, +`CONFIG_YAML`, `DEBUG`. ## Betrieb (Docker) @@ -33,6 +33,76 @@ docker compose up -d --build docker compose logs -f ``` +## Gundi-Push-Pfad (`gundi-receiver`) + +ICU pusht Events ueber eine Gundi-v2-kompatible API. Der Service `gundi-receiver` +(gleiches Image, `gundi_receiver.py`) nimmt sie entgegen: + +| Route | Zweck | +|-------|-------| +| `POST /v2/events/` | `camera_status_report` oder `cameratrap_rep` (JSON, Objekt oder Liste) | +| `POST /v2/events//attachments/` | JPEG zum `cameratrap_rep` (multipart/form-data) | +| `GET /healthz` | Healthcheck | + +Auth ueber den Header `apikey` = `GUNDI_API_KEY` aus der `.env`. + +Ablauf: Event und JPEG werden durable abgelegt und sofort quittiert. Stufe 1 +(Sekunden) schreibt processed-JPEG + Thumbnail und `image` + `detection`. Stufe 2 +liest danach asynchron per OCR den Kamera-Zeitstempel, setzt `image.ocr_ts` und +`DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert +Events ohne Bild nach `abandon_after_hours` als `ABANDONED`. + +### Datenmodell (`schema/001_domain_model.sql`) + +| Tabelle | Inhalt | +|---------|--------| +| `remote_cam.camera_status` | Statusupdates / Kameraeigenschaften (Zeitreihe je IMEI) | +| `remote_cam.image` | Bildeigenschaften inkl. `servertime`, `ocr_ts`, `ocr_state` | +| `remote_cam.detection` | Objektklassifikationen, `confidence` 0..1 | +| `remote_cam.image_with_status` | View: Bild + zum `servertime` gueltiger Status | +| `remote_cam.gundi_inbox` | Transportschicht: Rohevents + Zustand des Push-Pfads | + +Das Schema legt der Receiver beim Start an. Die Legacy-Tabellen `metadata` / +`category` werden vom alten Poller weiter befuellt, bis das Portal umgestellt ist. + +### Bestandsdaten migrieren + +Split `metadata` -> `image` + `camera_status`, `category` -> `detection`. +Wiederholbar, batchweise, liest die Alttabellen nur: + +```bash +docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy +``` + +Vor dem Portal-Cutover erneut ausfuehren, um zwischenzeitlich neue Bilder +nachzuziehen. `--verify-only` prueft nur die Zaehlungen. + +### Test + +```bash +docker compose up -d --build gundi-receiver +docker compose run --rm --no-deps gundi-receiver \ + python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg /pfad/zu/bild.jpg +``` + +Standardmaessig schreibt der Pfad unter `icu/gundi-test/` (siehe `gundi:` in +`config.yaml`); fuer den Produktivbetrieb `processed_prefix` / `thumb_prefix` auf +`icu/processed/` / `icu/thumbnails/` umstellen. + +### nginx (`icu.meles.eu`) + +vhost liegt unter [`deploy/nginx/icu.meles.eu`](deploy/nginx/icu.meles.eu): + +```bash +sudo cp deploy/nginx/icu.meles.eu /etc/nginx/sites-available/ +sudo ln -s /etc/nginx/sites-available/icu.meles.eu /etc/nginx/sites-enabled/ +sudo nginx -t && sudo systemctl reload nginx +sudo certbot --nginx -d icu.meles.eu --redirect +``` + +ICU-Endpunkt: `https://icu.meles.eu/v2/events/` (immer https — der 80->443-Redirect +ist ein 301 und wuerde POST in GET verwandeln). + ## Sicherheitshinweise - **Niemals** Secrets in `config.yaml` oder ins Git schreiben - nur in `.env`. diff --git a/config.yaml b/config.yaml index 8813e01..e84758d 100644 --- a/config.yaml +++ b/config.yaml @@ -39,3 +39,20 @@ app: enable_deermapper_api: false deermapper_api_timeout_seconds: 20 deermapper_api_image_field: "image" + +# Gundi-v2-Push-Empfaenger (Service gundi-receiver). +# api key -> via ENV (.env): GUNDI_API_KEY (min. 24 Zeichen) +gundi: + enabled: true + bind_host: "0.0.0.0" # im Container; nach aussen nur 127.0.0.1:8090 (compose) + bind_port: 8090 + inbox_prefix: "icu/gundi-test/inbox/" + processed_prefix: "icu/gundi-test/processed/" # spaeter: icu/processed/ + thumb_prefix: "icu/gundi-test/thumbnails/" # spaeter: icu/thumbnails/ + workers: 4 + ocr_workers: 2 + max_request_bytes: 33554432 + abandon_after_hours: 24 + sweep_seconds: 60 + stale_after_seconds: 300 + max_attempts: 3 diff --git a/deploy/nginx/icu.meles.eu b/deploy/nginx/icu.meles.eu new file mode 100644 index 0000000..68de124 --- /dev/null +++ b/deploy/nginx/icu.meles.eu @@ -0,0 +1,24 @@ +# nginx-vhost fuer den Gundi-Receiver. Nur HTTP; TLS ergaenzt certbot --nginx. +# ICU muss direkt https:// verwenden: der 80->443-Redirect von certbot ist ein 301 +# und wuerde POST in GET verwandeln. +server { + listen 80; + server_name icu.meles.eu; + + client_max_body_size 32m; # = gundi.max_request_bytes + + location /v2/events { + proxy_pass http://127.0.0.1:8090; + include /etc/nginx/proxy_params; + proxy_read_timeout 60s; + } + + location = /healthz { + proxy_pass http://127.0.0.1:8090; + include /etc/nginx/proxy_params; + } + + location / { + return 404; + } +} diff --git a/docker-compose.yaml b/docker-compose.yaml index 88d89fa..2876bda 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,3 +18,25 @@ services: # 2) folgende Zeilen einkommentieren - bindet debugpy NUR an localhost: # ports: # - "127.0.0.1:5678:5678" + + # Gundi-v2-Push-Empfaenger. Gleiches Image, eigener Prozess: startet und + # stirbt unabhaengig vom Poller. Nur an localhost gebunden - TLS und + # oeffentliche Erreichbarkeit uebernimmt nginx (icu.meles.eu). + gundi-receiver: + build: . + container_name: melesicumover-gundi + restart: unless-stopped + command: ["python", "/app/gundi_receiver.py"] + env_file: + - .env + environment: + CONFIG_YAML: /app/config.yaml + DEBUG: "0" + ports: + - "127.0.0.1:8090:8090" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/healthz', timeout=3)"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s diff --git a/gundi_pipeline.py b/gundi_pipeline.py new file mode 100644 index 0000000..cc2b295 --- /dev/null +++ b/gundi_pipeline.py @@ -0,0 +1,646 @@ +"""Push-Pfad (Gundi v2): Mapping ins Fachmodell, Ingest (Stufe 1), OCR (Stufe 2), Sweep.""" +import os +import re +import subprocess +import tempfile +import threading +import time +import uuid as uuidlib +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import psycopg +import yaml +from dateutil import tz +from psycopg.types.json import Json + +from main import ( + AppConfig, + S3, + exif_write_with_exiftool, + make_thumbnail_bytes, + ocr_extract_timestamp, + to_exif_datetime, +) + +SCHEMA_FILE = Path(__file__).resolve().parent / "schema" / "001_domain_model.sql" +SCHEMA_LOCK_KEY = 7_311_000_001 +UUID_STRICT_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + +EVENT_CAMERATRAP = "cameratrap_rep" +EVENT_STATUS = "camera_status_report" +MIN_API_KEY_LEN = 24 + + +# ----------------------- +# Config +# ----------------------- +@dataclass +class GundiConfig: + enabled: bool + api_key: str + bind_host: str = "0.0.0.0" + bind_port: int = 8090 + inbox_prefix: str = "icu/gundi-test/inbox/" + processed_prefix: str = "icu/gundi-test/processed/" + thumb_prefix: str = "icu/gundi-test/thumbnails/" + workers: int = 4 + ocr_workers: int = 2 + max_request_bytes: int = 32 * 1024 * 1024 + abandon_after_hours: int = 24 + sweep_seconds: int = 60 + stale_after_seconds: int = 300 + max_attempts: int = 3 + + +def load_gundi_config(path: str) -> GundiConfig: + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + g = raw.get("gundi") or {} + defaults = GundiConfig(enabled=False, api_key="") + + def prefix(key: str) -> str: + value = str(g.get(key, getattr(defaults, key))) + return value if value.endswith("/") else value + "/" + + gcfg = GundiConfig( + enabled=bool(g.get("enabled", False)), + api_key=os.environ.get("GUNDI_API_KEY", ""), + bind_host=str(g.get("bind_host", defaults.bind_host)), + bind_port=int(g.get("bind_port", defaults.bind_port)), + inbox_prefix=prefix("inbox_prefix"), + processed_prefix=prefix("processed_prefix"), + thumb_prefix=prefix("thumb_prefix"), + workers=max(1, int(g.get("workers", defaults.workers))), + ocr_workers=max(1, int(g.get("ocr_workers", defaults.ocr_workers))), + max_request_bytes=int(g.get("max_request_bytes", defaults.max_request_bytes)), + abandon_after_hours=int(g.get("abandon_after_hours", defaults.abandon_after_hours)), + sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)), + stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)), + max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))), + ) + if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN: + raise SystemExit( + f"GUNDI_API_KEY fehlt oder ist kuerzer als {MIN_API_KEY_LEN} Zeichen " + "(gundi.enabled ist aktiv). Bitte in der .env setzen." + ) + return gcfg + + +def connect(dsn: str) -> psycopg.Connection: + return psycopg.connect(dsn, connect_timeout=10) + + +def apply_schema(dsn: str) -> None: + sql = SCHEMA_FILE.read_text(encoding="utf-8") + with connect(dsn) as conn, conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,)) + cur.execute(sql) + + +def utc_now_str() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +# ----------------------- +# Validierung + Mapping +# ----------------------- +class EventError(ValueError): + pass + + +def _num(v) -> Optional[float]: + if isinstance(v, bool) or not isinstance(v, (int, float)): + return None + return float(v) + + +def _int(v) -> Optional[int]: + if isinstance(v, bool): + return None + if isinstance(v, int): + return v + if isinstance(v, float) and v.is_integer(): + return int(v) + if isinstance(v, str) and re.fullmatch(r"-?\d+", v.strip()): + return int(v.strip()) + return None + + +def _bool(v) -> Optional[bool]: + if isinstance(v, bool): + return v + if isinstance(v, str) and v.strip().lower() in ("true", "false"): + return v.strip().lower() == "true" + return None + + +def _str(v) -> Optional[str]: + if v is None: + return None + s = str(v).strip() + return s or None + + +def parse_ts(value) -> datetime: + if not isinstance(value, str) or not value.strip(): + raise EventError("recorded_at fehlt") + s = value.strip() + if s.endswith("Z"): + s = s[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(s) + except ValueError: + raise EventError(f"recorded_at ungueltig: {value!r}") from None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def coords(location) -> tuple[Optional[float], Optional[float]]: + """ICU meldet unbekannte Koordinaten als 0.0/0.0 -> NULL.""" + if not isinstance(location, dict): + return None, None + lat, lon = _num(location.get("lat")), _num(location.get("lon")) + if lat is None or lon is None or (lat == 0 and lon == 0): + return None, None + return lat, lon + + +def validate_event(ev) -> dict: + if not isinstance(ev, dict): + raise EventError("Event muss ein JSON-Objekt sein") + event_type = ev.get("event_type") + if not isinstance(event_type, str) or not event_type: + raise EventError("event_type fehlt") + source = ev.get("source") + if not isinstance(source, str) or not source.strip(): + raise EventError("source fehlt") + details = ev.get("event_details") + if details is None: + details = {} + if not isinstance(details, dict): + raise EventError("event_details muss ein Objekt sein") + if event_type == EVENT_CAMERATRAP: + dets = details.get("detections", []) + if not isinstance(dets, list) or not all(isinstance(d, dict) for d in dets): + raise EventError("event_details.detections muss eine Liste von Objekten sein") + lat, lon = coords(ev.get("location")) + return { + "event_type": event_type, + "source": source.strip(), + "recorded_at": parse_ts(ev.get("recorded_at")), + "lat": lat, + "lon": lon, + "details": details, + } + + +def camera_status_row(env: dict, object_id: str, raw: dict) -> dict: + d = env["details"] + return { + "imei": env["source"], + "reported_at": env["recorded_at"], + "iccid": _str(d.get("iccid")), + "init_key": _str(d.get("init_key")), + "camera_name": _str(d.get("camera_name")), + "camera_model": _str(d.get("camera_model")), + "firmware_version": _str(d.get("firmware_version")), + "camera_language": _str(d.get("camera_language")), + "customer_timezone": _str(d.get("customer_timezone")), + "mcc": _str(d.get("mobile_country_code")), + "mnc": _str(d.get("mobile_network_code")), + "lac": _str(d.get("location_area_code")), + "cell_id": _str(d.get("cell_id")), + "battery_pct": _int(d.get("battery_status")), + "signal_strength": _int(d.get("signal_strength")), + "temperature_c": _int(d.get("temperature")), + "coordinate_switch": _bool(d.get("coordinate_switch")), + "work_period": _bool(d.get("work_period")), + "work_start": _str(d.get("work_start")), + "work_end": _str(d.get("work_end")), + "image_format": _str(d.get("image_format")), + "image_night": _str(d.get("image_night")), + "thumbnail_size": _str(d.get("thumbnail_size")), + "pir_interval": _str(d.get("pir_interval")), + "time_scan": _str(d.get("time_scan")), + "lat": env["lat"], + "lon": env["lon"], + "source": "gundi", + "source_ref": object_id, + "raw": Json(raw), + } + + +def detection_rows(env: dict, image_uuid: str) -> list[dict]: + rows = [] + for det in env["details"].get("detections") or []: + category = _str(det.get("category")) + if category is None: + continue + pct = _num(det.get("confidence_percent")) + confidence = None if pct is None else min(max(pct / 100.0, 0.0), 1.0) + bbox = [_int(det.get(k)) for k in ("x_px", "y_px", "width_px", "height_px")] + if all(v in (None, 0) for v in bbox): + bbox = [None, None, None, None] + rows.append({ + "image_uuid": image_uuid, + "category": category, + "confidence": confidence, + "bbox_x": bbox[0], + "bbox_y": bbox[1], + "bbox_w": bbox[2], + "bbox_h": bbox[3], + "model_version": "icu", + "source": "gundi", + }) + return rows + + +def image_row(env: dict, object_id: str, raw: dict, size_bytes: int, processed_key: str, thumb_key: str) -> dict: + d = env["details"] + return { + "image_uuid": object_id, + "imei": env["source"], + "servertime": env["recorded_at"], + "ocr_state": "PENDING", + "camera_name": _str(d.get("cameratraprep_camera-name")), + "camera_make": _str(d.get("cameratraprep_camera-make")), + "lat": env["lat"], + "lon": env["lon"], + "size_bytes": size_bytes, + "processed_key": processed_key, + "thumbnail_key": thumb_key, + "source": "gundi", + "source_ref": object_id, + "raw": Json(raw), + } + + +def _insert(cur, table: str, row: dict, suffix: str = "") -> int: + cols = list(row) + cur.execute( + f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) {suffix}", + [row[c] for c in cols], + ) + return cur.rowcount + + +# ----------------------- +# Receiver-seitige Persistenz (synchron im Request) +# ----------------------- +def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]: + """Ein Request = eine Transaktion. Antwort erst nach Commit.""" + results = [] + with connect(dsn) as conn, conn.cursor() as cur: + for env, raw in items: + object_id = str(uuidlib.uuid4()) + if env["event_type"] == EVENT_STATUS: + state = "INGESTED" + elif env["event_type"] == EVENT_CAMERATRAP: + state = "AWAITING_ATTACHMENT" + else: + state = "IGNORED" + _insert(cur, "remote_cam.gundi_inbox", { + "object_id": object_id, + "event_type": env["event_type"], + "source": env["source"], + "recorded_at": env["recorded_at"], + "payload": Json(raw), + "state": state, + }) + if env["event_type"] == EVENT_STATUS: + _insert( + cur, "remote_cam.camera_status", camera_status_row(env, object_id, raw), + "ON CONFLICT (imei, reported_at) WHERE source = 'gundi' DO NOTHING", + ) + results.append({ + "object_id": object_id, + "created_at": utc_now_str(), + "event_type": env["event_type"], + "source": env["source"], + "state": state, + }) + return results + + +def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str: + """Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.""" + with connect(dsn) as conn, conn.cursor() as cur: + cur.execute( + "SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE", + (object_id,), + ) + row = cur.fetchone() + if not row or row[0] != EVENT_CAMERATRAP: + raise LookupError(object_id) + if row[1] in ("RECEIVED", "INGESTED"): + return "duplicate" + key = f"{gcfg.inbox_prefix}{object_id}.jpg" + s3.put_bytes(key, jpg, "image/jpeg") + cur.execute( + """ + UPDATE remote_cam.gundi_inbox + SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(), + attempts = 0, last_error = NULL, updated_at = now() + WHERE object_id = %s + """, + (key, object_id), + ) + return "stored" + + +# ----------------------- +# EXIF +# ----------------------- +def exif_set_digitized(jpg: bytes, servertime: datetime, *, timeout_seconds: int) -> bytes: + """Stufe 1: nur Serverzeit. DateTimeOriginal folgt nach dem OCR (Stufe 2).""" + dtd = to_exif_datetime(servertime.astimezone(timezone.utc)) + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "in.jpg") + with open(path, "wb") as f: + f.write(jpg) + subprocess.run( + [ + "exiftool", "-overwrite_original", + f"-DateTimeDigitized={dtd}", "-OffsetTimeDigitized=+00:00", + f"-ModifyDate={dtd}", "-OffsetTime=+00:00", + path, + ], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_seconds, + ) + with open(path, "rb") as f: + return f.read() + + +# ----------------------- +# Pipeline +# ----------------------- +class Pipeline: + def __init__(self, cfg: AppConfig, gcfg: GundiConfig): + self.cfg = cfg + self.gcfg = gcfg + self.s3 = S3(cfg) + self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest") + self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr") + self._stop = threading.Event() + + def start(self) -> None: + threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start() + + def stop(self) -> None: + self._stop.set() + self._ingest_pool.shutdown(wait=False, cancel_futures=True) + self._ocr_pool.shutdown(wait=False, cancel_futures=True) + + def submit_ingest(self, object_id: str) -> None: + self._ingest_pool.submit(self._guarded, self.ingest, object_id) + + def submit_ocr(self, image_uuid: str) -> None: + self._ocr_pool.submit(self._guarded, self.ocr, image_uuid) + + @staticmethod + def _guarded(fn, arg: str) -> None: + try: + fn(arg) + except Exception as e: + print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True) + + # --- Stufe 1 --------------------------------------------------------- + def ingest(self, object_id: str) -> None: + start = time.perf_counter() + try: + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT payload, attachment_key FROM remote_cam.gundi_inbox + WHERE object_id = %s AND state = 'RECEIVED' + FOR UPDATE SKIP LOCKED + """, + (object_id,), + ) + row = cur.fetchone() + if not row: + return + payload, attachment_key = row + env = validate_event(payload) + jpg = self.s3.get_bytes(attachment_key) + + exif_note = "ok" + try: + processed = exif_set_digitized( + jpg, env["recorded_at"], timeout_seconds=self.cfg.exiftool_timeout_seconds + ) + except Exception as e: + processed = jpg + exif_note = f"fail({type(e).__name__})" + thumb = make_thumbnail_bytes(jpg, self.cfg.thumb_max) + + processed_key = f"{self.gcfg.processed_prefix}{object_id}.jpg" + thumb_key = f"{self.gcfg.thumb_prefix}{object_id}.jpg" + self.s3.put_bytes(processed_key, processed, "image/jpeg") + self.s3.put_bytes(thumb_key, thumb, "image/jpeg") + + n_det = 0 + inserted = _insert( + cur, "remote_cam.image", + image_row(env, object_id, payload, len(jpg), processed_key, thumb_key), + "ON CONFLICT (image_uuid) DO NOTHING", + ) + if inserted: + for det in detection_rows(env, object_id): + n_det += _insert(cur, "remote_cam.detection", det) + cur.execute( + """ + UPDATE remote_cam.gundi_inbox + SET state = 'INGESTED', last_error = NULL, updated_at = now() + WHERE object_id = %s + """, + (object_id,), + ) + except Exception as e: + self._record_ingest_failure(object_id, e) + return + + try: + self.s3.delete_keys([attachment_key]) + except Exception as e: + print(f"[GUNDI] inbox-cleanup {object_id} fehlgeschlagen: {e}", flush=True) + self.submit_ocr(object_id) + ms = int((time.perf_counter() - start) * 1000) + print( + f"[GUNDI] ingest object_id={object_id} imei={env['source']} detections={n_det} " + f"exif={exif_note} ms={ms}", + flush=True, + ) + + def _record_ingest_failure(self, object_id: str, err: Exception) -> None: + msg = f"{type(err).__name__}: {err}"[:1000] + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + UPDATE remote_cam.gundi_inbox + SET attempts = attempts + 1, + state = CASE WHEN attempts + 1 >= %s THEN 'FAILED' ELSE state END, + last_error = %s, updated_at = now() + WHERE object_id = %s AND state = 'RECEIVED' + RETURNING attempts, state + """, + (self.gcfg.max_attempts, msg, object_id), + ) + row = cur.fetchone() + if row: + print(f"[GUNDI] ingest object_id={object_id} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True) + + # --- Stufe 2 --------------------------------------------------------- + def ocr(self, image_uuid: str) -> None: + start = time.perf_counter() + try: + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT imei, servertime, processed_key FROM remote_cam.image + WHERE image_uuid = %s AND ocr_state = 'PENDING' + FOR UPDATE SKIP LOCKED + """, + (image_uuid,), + ) + row = cur.fetchone() + if not row: + return + imei, servertime, processed_key = row + tzinfo, tzname = self._camera_tz(cur, imei, servertime) + jpg = self.s3.get_bytes(processed_key) + ocr_ts = ocr_extract_timestamp( + jpg, self.cfg.ocr_crop_w_frac, self.cfg.ocr_crop_h_frac, tzinfo + ) + + exif_err = None + try: + patched = exif_write_with_exiftool( + jpg, ocr_ts, servertime, timeout_seconds=self.cfg.exiftool_timeout_seconds + ) + self.s3.put_bytes(processed_key, patched, "image/jpeg") + except Exception as e: + exif_err = f"exif: {type(e).__name__}: {e}"[:1000] + + cur.execute( + """ + UPDATE remote_cam.image + SET ocr_ts = %s, ocr_state = 'OK', ocr_attempts = ocr_attempts + 1, + ocr_error = %s, updated_at = now() + WHERE image_uuid = %s + """, + (ocr_ts, exif_err, image_uuid), + ) + except Exception as e: + self._record_ocr_failure(image_uuid, e) + return + + drift = int((servertime - ocr_ts).total_seconds()) + ms = int((time.perf_counter() - start) * 1000) + print( + f"[GUNDI] ocr image_uuid={image_uuid} ocr_ts={ocr_ts.isoformat()} tz={tzname} " + f"servertime-ocr={drift}s exif={'ok' if exif_err is None else 'fail'} ms={ms}", + flush=True, + ) + + @staticmethod + def _camera_tz(cur, imei: str, servertime: datetime): + cur.execute( + """ + SELECT customer_timezone FROM remote_cam.camera_status + WHERE imei = %s AND customer_timezone IS NOT NULL AND reported_at <= %s + ORDER BY reported_at DESC LIMIT 1 + """, + (imei, servertime), + ) + row = cur.fetchone() + if not row: + cur.execute( + """ + SELECT customer_timezone FROM remote_cam.camera_status + WHERE imei = %s AND customer_timezone IS NOT NULL + ORDER BY reported_at ASC LIMIT 1 + """, + (imei,), + ) + row = cur.fetchone() + name = row[0] if row else "UTC" + zone = tz.gettz(name) + if zone is None: + return timezone.utc, "UTC" + return zone, name + + def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None: + msg = f"{type(err).__name__}: {err}"[:1000] + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + UPDATE remote_cam.image + SET ocr_attempts = ocr_attempts + 1, + ocr_state = CASE WHEN ocr_attempts + 1 >= %s THEN 'FAIL' ELSE ocr_state END, + ocr_error = %s, updated_at = now() + WHERE image_uuid = %s AND ocr_state = 'PENDING' + RETURNING ocr_attempts, ocr_state + """, + (self.gcfg.max_attempts, msg, image_uuid), + ) + row = cur.fetchone() + if row: + print(f"[GUNDI] ocr image_uuid={image_uuid} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True) + + # --- Sweep ----------------------------------------------------------- + def _sweep_loop(self) -> None: + delay = 0 + while not self._stop.wait(delay): + delay = self.gcfg.sweep_seconds + try: + self.sweep() + except Exception as e: + print(f"[GUNDI] sweep Fehler: {type(e).__name__}: {e}", flush=True) + + def sweep(self) -> None: + stale = self.gcfg.stale_after_seconds + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT object_id::text FROM remote_cam.gundi_inbox + WHERE state = 'RECEIVED' AND updated_at < now() - make_interval(secs => %s) + ORDER BY updated_at LIMIT 200 + """, + (stale,), + ) + ingest_ids = [r[0] for r in cur.fetchall()] + cur.execute( + """ + SELECT image_uuid::text FROM remote_cam.image + WHERE ocr_state = 'PENDING' AND source = 'gundi' + AND updated_at < now() - make_interval(secs => %s) + ORDER BY updated_at LIMIT 200 + """, + (stale,), + ) + ocr_ids = [r[0] for r in cur.fetchall()] + cur.execute( + """ + UPDATE remote_cam.gundi_inbox + SET state = 'ABANDONED', updated_at = now() + WHERE state = 'AWAITING_ATTACHMENT' + AND received_at < now() - make_interval(hours => %s) + """, + (self.gcfg.abandon_after_hours,), + ) + abandoned = cur.rowcount + for object_id in ingest_ids: + self.submit_ingest(object_id) + for image_uuid in ocr_ids: + self.submit_ocr(image_uuid) + if ingest_ids or ocr_ids or abandoned: + print( + f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}", + flush=True, + ) diff --git a/gundi_receiver.py b/gundi_receiver.py new file mode 100644 index 0000000..999d49e --- /dev/null +++ b/gundi_receiver.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Gundi-v2-kompatibler Empfaenger fuer ICU-Push-Events. + + POST /v2/events/ -> {"object_id", "created_at"} + POST /v2/events//attachments/ -> {"object_id", "created_at"} + GET /healthz +""" +import hmac +import json +import os +import re +import signal +import threading +import time +from email.parser import BytesParser +from email.policy import default as email_policy +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +from gundi_pipeline import ( + UUID_STRICT_RE, + EventError, + GundiConfig, + Pipeline, + apply_schema, + attach_image, + load_gundi_config, + store_events, + utc_now_str, + validate_event, +) +from main import AppConfig, load_config + +EVENTS_PATH = "/v2/events/" +ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$") +JPEG_MAGIC = b"\xff\xd8\xff" + + +def extract_jpeg(content_type: str, body: bytes) -> tuple[bytes | None, int]: + """Erstes JPEG aus einem multipart/form-data-Body; Client-Dateinamen werden ignoriert.""" + raw = f"Content-Type: {content_type}\r\nMIME-Version: 1.0\r\n\r\n".encode("latin-1") + body + msg = BytesParser(policy=email_policy).parsebytes(raw) + if not msg.is_multipart(): + return None, 0 + files = [] + for part in msg.iter_parts(): + if part.get_content_disposition() != "form-data" or not part.get_filename(): + continue + files.append(part.get_payload(decode=True) or b"") + jpgs = [f for f in files if f.startswith(JPEG_MAGIC)] + return (jpgs[0] if jpgs else None), len(files) + + +class GundiServer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, addr, cfg: AppConfig, gcfg: GundiConfig, pipeline: Pipeline): + super().__init__(addr, GundiHandler) + self.cfg = cfg + self.gcfg = gcfg + self.pipeline = pipeline + + +class GundiHandler(BaseHTTPRequestHandler): + server: GundiServer + server_version = "melesICUmover-gundi/1.0" + sys_version = "" + timeout = 60 + + def log_message(self, fmt, *args): + client = self.headers.get("X-Real-IP") if self.headers else None + print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True) + + # --- Antworten ------------------------------------------------------- + def _json(self, code: int, data) -> None: + body = json.dumps(data, ensure_ascii=False).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _error(self, code: int, message: str, *, close: bool = False) -> None: + if close: + self.close_connection = True + self._json(code, {"error": message}) + + def _redirect(self, location: str) -> None: + # 308 statt 301: der Client muss POST + Body beibehalten. + self.close_connection = True + self.send_response(308) + self.send_header("Location", location) + self.send_header("Content-Length", "0") + self.end_headers() + + # --- Request-Helfer -------------------------------------------------- + def _authorized(self) -> bool: + got = (self.headers.get("apikey") or "").encode("utf-8") + return hmac.compare_digest(got, self.server.gcfg.api_key.encode("utf-8")) + + def _read_body(self) -> bytes | None: + if self.headers.get("Transfer-Encoding"): + self._error(411, "Transfer-Encoding wird nicht unterstuetzt, Content-Length senden", close=True) + return None + raw_len = self.headers.get("Content-Length") + if raw_len is None: + self._error(411, "Content-Length erforderlich", close=True) + return None + try: + length = int(raw_len) + except ValueError: + self._error(400, "Content-Length ungueltig", close=True) + return None + if length < 0: + self._error(400, "Content-Length ungueltig", close=True) + return None + if length > self.server.gcfg.max_request_bytes: + self._error(413, "Request zu gross", close=True) + return None + body = self.rfile.read(length) + if len(body) != length: + self._error(400, "Body unvollstaendig", close=True) + return None + return body + + # --- Routing --------------------------------------------------------- + def do_GET(self) -> None: + if urlparse(self.path).path == "/healthz": + self._json(200, {"status": "ok"}) + else: + self._error(404, "not found") + + def do_POST(self) -> None: + path = urlparse(self.path).path + if path == "/v2/events": + self._redirect(EVENTS_PATH) + return + attachment = ATTACHMENT_RE.match(path) + if attachment and not attachment.group(2): + self._redirect(path + "/") + return + if path != EVENTS_PATH and not attachment: + self._error(404, "not found", close=True) + return + if not self._authorized(): + self._error(401, "Invalid or missing apikey", close=True) + return + body = self._read_body() + if body is None: + return + try: + if attachment: + self._handle_attachment(attachment.group(1).lower(), body) + else: + self._handle_events(body) + except Exception as e: + print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True) + self._error(500, "internal error", close=True) + + # --- Handler --------------------------------------------------------- + def _handle_events(self, body: bytes) -> None: + if "application/json" not in (self.headers.get("Content-Type") or "").lower(): + self._error(415, "Content-Type muss application/json sein") + return + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + self._error(400, "Ungueltiges JSON") + return + events = payload if isinstance(payload, list) else [payload] + if not events: + self._error(400, "Leere Event-Liste") + return + try: + envs = [validate_event(ev) for ev in events] + except EventError as e: + self._error(400, str(e)) + return + + results = store_events(self.server.cfg.pg_dsn, list(zip(envs, events))) + for r in results: + print( + f"[GUNDI] event object_id={r['object_id']} type={r['event_type']} " + f"imei={r['source']} state={r['state']}", + flush=True, + ) + out = [{"object_id": r["object_id"], "created_at": r["created_at"]} for r in results] + self._json(200, out if isinstance(payload, list) else out[0]) + + def _handle_attachment(self, object_id: str, body: bytes) -> None: + if not UUID_STRICT_RE.match(object_id): + self._error(404, "Unknown object_id") + return + content_type = self.headers.get("Content-Type") or "" + if "multipart/form-data" not in content_type.lower(): + self._error(415, "Content-Type muss multipart/form-data sein") + return + jpg, n_files = extract_jpeg(content_type, body) + if jpg is None: + self._error(400, "Kein JPEG-Anhang gefunden") + return + try: + outcome = attach_image( + self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg + ) + except LookupError: + self._error(404, "Unknown object_id") + return + if outcome == "stored": + self.server.pipeline.submit_ingest(object_id) + print( + f"[GUNDI] attachment object_id={object_id} bytes={len(jpg)} files={n_files} outcome={outcome}", + flush=True, + ) + self._json(200, {"object_id": object_id, "created_at": utc_now_str()}) + + +def main() -> None: + cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml") + cfg = load_config(cfg_path) + gcfg = load_gundi_config(cfg_path) + if not gcfg.enabled: + print("[GUNDI] gundi.enabled ist false - Receiver bleibt inaktiv.", flush=True) + while True: + time.sleep(3600) + + apply_schema(cfg.pg_dsn) + pipeline = Pipeline(cfg, gcfg) + pipeline.start() + server = GundiServer((gcfg.bind_host, gcfg.bind_port), cfg, gcfg, pipeline) + + def _shutdown(signum, frame): + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, _shutdown) + signal.signal(signal.SIGINT, _shutdown) + + print( + f"[GUNDI] lausche auf {gcfg.bind_host}:{gcfg.bind_port} " + f"inbox={gcfg.inbox_prefix} processed={gcfg.processed_prefix} thumbs={gcfg.thumb_prefix}", + flush=True, + ) + try: + server.serve_forever() + finally: + server.server_close() + pipeline.stop() + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py index d28ba0a..e7504e9 100644 --- a/main.py +++ b/main.py @@ -25,14 +25,6 @@ import pytesseract import yaml -if os.environ.get("DEBUG") == "1": - import debugpy - debugpy.listen(("0.0.0.0", 5678)) - print("Waiting for debugger...") - debugpy.wait_for_client() - - - # ----------------------- # Config # ----------------------- @@ -1012,6 +1004,12 @@ def run_once(cfg: AppConfig) -> int: def main(): + if os.environ.get("DEBUG") == "1": + import debugpy + debugpy.listen(("0.0.0.0", 5678)) + print("Waiting for debugger...") + debugpy.wait_for_client() + cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml") cfg = load_config(cfg_path) diff --git a/schema/001_domain_model.sql b/schema/001_domain_model.sql new file mode 100644 index 0000000..4ff592b --- /dev/null +++ b/schema/001_domain_model.sql @@ -0,0 +1,179 @@ +-- Normalisiertes Fachmodell: Kamerastatus, Bildeigenschaften, Objektklassifikationen. +-- Idempotent; wird vom gundi-receiver beim Start unter Advisory-Lock angewendet. +-- Setzt das bestehende Schema remote_cam voraus (die App-Rolle hat kein CREATE auf DB-Ebene). + +-- Statusupdates / Kameraeigenschaften: eine Zeile je Statusmeldung (Zeitreihe). +CREATE TABLE IF NOT EXISTS remote_cam.camera_status ( + status_id bigserial PRIMARY KEY, + imei text NOT NULL, + reported_at timestamptz NOT NULL, + received_at timestamptz NOT NULL DEFAULT now(), + iccid text, + init_key text, + camera_name text, + camera_model text, + firmware_version text, + camera_language text, + customer_timezone text, + mcc text, + mnc text, + lac text, + cell_id text, + camera_network text, + battery_pct integer, + signal_strength integer, + temperature_c integer, + coordinate_switch boolean, + work_period boolean, + work_start text, + work_end text, + image_format text, + image_night text, + thumbnail_size text, + pir_interval text, + time_scan text, + lat double precision, + lon double precision, + settings_changed_at timestamptz, + source text NOT NULL CHECK (source IN ('gundi', 'entrance')), + source_ref text, + raw jsonb +); + +-- Legacy hat 173 doppelte (imei, servertime)-Paare, daher keine globale Eindeutigkeit. +CREATE UNIQUE INDEX IF NOT EXISTS camera_status_gundi_uq + ON remote_cam.camera_status (imei, reported_at) WHERE source = 'gundi'; +CREATE UNIQUE INDEX IF NOT EXISTS camera_status_entrance_uq + ON remote_cam.camera_status (source_ref) WHERE source = 'entrance'; +CREATE INDEX IF NOT EXISTS camera_status_imei_reported_idx + ON remote_cam.camera_status (imei, reported_at DESC); + +-- Bildeigenschaften. +CREATE TABLE IF NOT EXISTS remote_cam.image ( + image_uuid uuid PRIMARY KEY, + imei text NOT NULL, + servertime timestamptz NOT NULL, + ocr_ts timestamptz, + ocr_state text NOT NULL DEFAULT 'PENDING' CHECK (ocr_state IN ('PENDING', 'OK', 'FAIL')), + ocr_attempts integer NOT NULL DEFAULT 0, + ocr_error text, + camera_ts timestamptz, + camera_name text, + camera_make text, + lat double precision, + lon double precision, + size_bytes integer, + processed_key text, + thumbnail_key text, + source text NOT NULL CHECK (source IN ('gundi', 'entrance')), + source_ref text, + imported_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + raw jsonb +); + +CREATE INDEX IF NOT EXISTS image_imei_servertime_idx + ON remote_cam.image (imei, servertime DESC); +CREATE INDEX IF NOT EXISTS image_servertime_idx + ON remote_cam.image (servertime DESC); +CREATE INDEX IF NOT EXISTS image_ocr_pending_idx + ON remote_cam.image (updated_at) WHERE ocr_state = 'PENDING'; + +-- Objektklassifikationen. +CREATE TABLE IF NOT EXISTS remote_cam.detection ( + detection_id bigserial PRIMARY KEY, + image_uuid uuid NOT NULL REFERENCES remote_cam.image (image_uuid) ON DELETE CASCADE, + category text NOT NULL, + confidence double precision CHECK (confidence BETWEEN 0 AND 1), + bbox_x integer, + bbox_y integer, + bbox_w integer, + bbox_h integer, + model_version text, + source text NOT NULL CHECK (source IN ('gundi', 'entrance')) +); + +CREATE INDEX IF NOT EXISTS detection_image_confidence_idx + ON remote_cam.detection (image_uuid, confidence DESC); +CREATE INDEX IF NOT EXISTS detection_category_idx + ON remote_cam.detection (category); + +-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine. +CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox ( + object_id uuid PRIMARY KEY, + event_type text NOT NULL, + source text, + recorded_at timestamptz, + payload jsonb NOT NULL, + received_at timestamptz NOT NULL DEFAULT now(), + attachment_key text, + attachment_received_at timestamptz, + state text NOT NULL CHECK (state IN ( + 'AWAITING_ATTACHMENT', 'RECEIVED', 'INGESTED', + 'FAILED', 'ABANDONED', 'IGNORED')), + attempts integer NOT NULL DEFAULT 0, + last_error text, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS gundi_inbox_open_idx + ON remote_cam.gundi_inbox (state, updated_at) + WHERE state IN ('AWAITING_ATTACHMENT', 'RECEIVED'); + +-- Bild + der zum Aufnahme-Serverzeitpunkt gueltige Kamerastatus. +-- Tie-Breaker bei gleichem reported_at: der aus derselben Legacy-Zeile abgespaltene Status. +CREATE OR REPLACE VIEW remote_cam.image_with_status AS +SELECT + i.image_uuid, + i.imei, + i.servertime, + i.ocr_ts, + i.ocr_state, + i.camera_ts, + i.camera_name, + i.camera_make, + i.lat, + i.lon, + i.size_bytes, + i.processed_key, + i.thumbnail_key, + i.source, + i.imported_at, + s.status_id, + s.reported_at AS status_reported_at, + s.iccid, + s.init_key, + s.camera_name AS status_camera_name, + s.camera_model, + s.firmware_version, + s.camera_language, + s.customer_timezone, + s.mcc, + s.mnc, + s.lac, + s.cell_id, + s.camera_network, + s.battery_pct, + s.signal_strength, + s.temperature_c, + s.coordinate_switch, + s.work_period, + s.work_start, + s.work_end, + s.image_format, + s.image_night, + s.thumbnail_size, + s.pir_interval, + s.time_scan, + s.settings_changed_at +FROM remote_cam.image i +LEFT JOIN LATERAL ( + SELECT cs.* + FROM remote_cam.camera_status cs + WHERE cs.imei = i.imei + AND cs.reported_at <= i.servertime + ORDER BY cs.reported_at DESC, + (cs.source_ref = i.image_uuid::text) DESC NULLS LAST, + cs.status_id DESC + LIMIT 1 +) s ON TRUE; diff --git a/schema/002_migrate_legacy.sql b/schema/002_migrate_legacy.sql new file mode 100644 index 0000000..b14fa6f --- /dev/null +++ b/schema/002_migrate_legacy.sql @@ -0,0 +1,94 @@ +-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt +-- ein image und ein camera_status; remote_cam.category wird zu detection. +-- +-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py +-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen. +-- Alttabellen werden nur gelesen. + +-- step: image +INSERT INTO remote_cam.image ( + image_uuid, imei, servertime, ocr_ts, ocr_state, ocr_error, camera_ts, + camera_name, camera_make, lat, lon, size_bytes, processed_key, thumbnail_key, + source, source_ref, imported_at, updated_at +) +SELECT + m.metadata_uuid::uuid, + m.imei, + m.servertime, + m.ocr_ts, + CASE WHEN m.ocr_ts IS NULL THEN 'FAIL' ELSE 'OK' END, + CASE WHEN m.ocr_ts IS NULL THEN 'legacy: no ocr_ts' END, + m."ImageCreationDate", + m."CameraName", + m."CameraModel", + CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Latitude" END, + CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Longitude" END, + m."SizeInByte", + j.processed_jpg_key, + j.thumbnail_key, + 'entrance', + m.metadata_uuid, + COALESCE(m.import_ts, now()), + now() +FROM _mig_batch b +JOIN remote_cam.metadata m ON m.metadata_uuid = b.metadata_uuid +LEFT JOIN remote_cam.import_job j ON j.image_uuid = m.metadata_uuid +WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL +ON CONFLICT (image_uuid) DO NOTHING; + +-- step: camera_status +INSERT INTO remote_cam.camera_status ( + imei, reported_at, received_at, iccid, init_key, camera_name, camera_model, + firmware_version, camera_language, customer_timezone, mcc, mnc, lac, camera_network, + battery_pct, signal_strength, temperature_c, coordinate_switch, work_period, + work_start, work_end, image_format, image_night, thumbnail_size, pir_interval, time_scan, + lat, lon, settings_changed_at, source, source_ref +) +SELECT + m.imei, + m.servertime, + COALESCE(m.import_ts, now()), + m.iccid, + m."INITKey", + m."CameraName", + m."CameraModel", + m."FirmwareVersion", + m."CameraLanguage", + m."CustomerTimezone", + m."MobileCountryCode", + m."MobileNetworkCode", + m."LocationAreaCode", + m."CameraNetwork", + m."BatteryStatus", + m."SignalStrength", + m."Temperature", + m."CoordinateSwitch", + m."WorkPeriod", + m."WorkStart", + m."WorkEnd", + m."ImageFormat", + m."ImageNight", + m."ThumbnailSize", + m."PIRInterval", + m."TimeScan", + CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Latitude" END, + CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Longitude" END, + m."DateTimeLastSettings", + 'entrance', + m.metadata_uuid +FROM _mig_batch b +JOIN remote_cam.metadata m ON m.metadata_uuid = b.metadata_uuid +WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL +ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING; + +-- step: detection +INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source) +SELECT i.image_uuid, c.category, c.score, c.version, 'entrance' +FROM _mig_batch b +JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid +JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid +WHERE c.category IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM remote_cam.detection d + WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance' + ); diff --git a/tools/gundi_smoke_test.py b/tools/gundi_smoke_test.py new file mode 100644 index 0000000..69fd101 --- /dev/null +++ b/tools/gundi_smoke_test.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Spielt die Beispiel-Payloads aus der ICU-Doku gegen den Gundi-Receiver. + + python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg bild.jpg + +Ohne --jpeg wird ein synthetisches Bild mit Zeitstempel-Overlay erzeugt (braucht Pillow). +Standard-IMEI ist eine Test-IMEI, damit echte Kamerazeitreihen unberuehrt bleiben. +""" +import argparse +import http.client +import io +import json +import os +import sys +import uuid +from datetime import datetime, timedelta, timezone +from urllib.parse import urlparse + +TEST_IMEI = "999000000000001" + + +def request(url: str, method: str, path: str, *, body: bytes = b"", headers: dict | None = None): + u = urlparse(url) + cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection + conn = cls(u.hostname, u.port, timeout=30) + try: + conn.request(method, path, body=body, headers=headers or {}) + resp = conn.getresponse() + data = resp.read() + return resp.status, dict(resp.getheaders()), data + finally: + conn.close() + + +def post_json(url, path, key, payload): + headers = {"Content-Type": "application/json"} + if key is not None: + headers["apikey"] = key + return request(url, "POST", path, body=json.dumps(payload).encode(), headers=headers) + + +def post_file(url, path, key, jpg: bytes): + boundary = f"----smoke-{uuid.uuid4().hex}" + body = ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file1"; filename="../../evil.jpg"\r\n' + f"Content-Type: image/jpeg\r\n\r\n" + ).encode() + jpg + f"\r\n--{boundary}--\r\n".encode() + return request(url, "POST", path, body=body, headers={ + "Content-Type": f"multipart/form-data; boundary={boundary}", + "apikey": key, + }) + + +def synthetic_jpeg(ts: datetime) -> bytes: + from PIL import Image, ImageDraw, ImageFont + + w, h = 1920, 1080 + im = Image.new("RGB", (w, h), (70, 90, 70)) + draw = ImageDraw.Draw(im) + bar = int(h * 0.05) + draw.rectangle((0, h - bar, w, h), fill=(0, 0, 0)) + font = ImageFont.load_default(size=int(bar * 0.8)) + text = ts.strftime("%H:%M:%S %d-%m-%Y") + tw = draw.textlength(text, font=font) + draw.text((w - tw - 20, h - bar + bar * 0.08), text, fill=(255, 255, 255), font=font) + out = io.BytesIO() + im.save(out, format="JPEG", quality=90) + return out.getvalue() + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--url", default="http://127.0.0.1:8090") + ap.add_argument("--key", default=os.environ.get("GUNDI_API_KEY")) + ap.add_argument("--imei", default=TEST_IMEI) + ap.add_argument("--jpeg", help="Pfad zu einem echten Kamerabild") + args = ap.parse_args() + if not args.key: + print("--key oder GUNDI_API_KEY fehlt", file=sys.stderr) + return 2 + + failures = 0 + + def check(label: str, ok: bool, detail: str = "") -> None: + nonlocal failures + failures += 0 if ok else 1 + print(f"[{'OK ' if ok else 'FAIL'}] {label}{(' - ' + detail) if detail else ''}") + + now = datetime.now(timezone.utc).replace(microsecond=0) + status_payload = { + "source": args.imei, + "event_type": "camera_status_report", + "title": "Camera Status report - Smoke Test", + "recorded_at": (now - timedelta(minutes=10)).isoformat().replace("+00:00", "Z"), + "location": {"lat": 0.0, "lon": 0.0}, + "event_details": { + "imei": args.imei, "iccid": "8900000000000000000", "init_key": "000000000000", + "camera_name": "Smoke Test", "camera_model": None, "firmware_version": "1.4.2", + "mobile_country_code": "232", "mobile_network_code": "01", "location_area_code": "1234", + "cell_id": "567890", "customer_timezone": "Europe/Vienna", "battery_status": 87, + "signal_strength": 3, "temperature": -4, "coordinate_switch": False, "work_period": True, + "image_format": "F4_3", "image_night": "MIN", "work_start": "06:00", "work_end": "21:00", + "thumbnail_size": "720p", "pir_interval": "30", "time_scan": "120", "camera_language": "en", + }, + } + trap_payload = { + "source": args.imei, + "event_type": "cameratrap_rep", + "title": "Camera trap image - Smoke Test", + "recorded_at": now.isoformat().replace("+00:00", "Z"), + "location": {"lat": 47.8095, "lon": 13.0550}, + "event_details": { + "cameratraprep_camera-name": "Smoke Test", + "cameratraprep_camera-make": "M350GW", + "detections": [ + {"category": "deer", "confidence_percent": 96.2, "x_px": 0, "y_px": 0, "width_px": 0, "height_px": 0}, + {"category": "human", "confidence_percent": 3.1, "x_px": 15, "y_px": 17, "width_px": 100, "height_px": 200}, + ], + }, + } + + code, _, _ = request(args.url, "GET", "/healthz") + check("healthz", code == 200, f"HTTP {code}") + + code, _, _ = post_json(args.url, "/v2/events/", None, status_payload) + check("ohne apikey -> 401", code == 401, f"HTTP {code}") + code, _, _ = post_json(args.url, "/v2/events/", "falsch", status_payload) + check("falscher apikey -> 401", code == 401, f"HTTP {code}") + + code, headers, _ = post_json(args.url, "/v2/events", args.key, status_payload) + check("ohne Slash -> 308", code == 308 and headers.get("Location") == "/v2/events/", + f"HTTP {code} Location={headers.get('Location')}") + + code, _, data = post_json(args.url, "/v2/events/", args.key, {"event_type": "cameratrap_rep"}) + check("unvollstaendiges Event -> 400", code == 400, f"HTTP {code} {data[:120]!r}") + + code, _, data = post_json(args.url, "/v2/events/", args.key, status_payload) + status_id = json.loads(data).get("object_id") if code == 200 else None + check("camera_status_report", code == 200 and bool(status_id), f"HTTP {code} object_id={status_id}") + code, _, _ = post_json(args.url, "/v2/events/", args.key, status_payload) + check("camera_status_report erneut (idempotent in camera_status)", code == 200, f"HTTP {code}") + + code, _, data = post_json(args.url, "/v2/events/", args.key, trap_payload) + object_id = json.loads(data).get("object_id") if code == 200 else None + check("cameratrap_rep", code == 200 and bool(object_id), f"HTTP {code} object_id={object_id}") + + if args.jpeg: + with open(args.jpeg, "rb") as f: + jpg = f.read() + else: + jpg = synthetic_jpeg(now.astimezone(timezone(timedelta(hours=2)))) + + code, _, _ = post_file(args.url, f"/v2/events/{uuid.uuid4()}/attachments/", args.key, jpg) + check("Attachment auf unbekannte object_id -> 404", code == 404, f"HTTP {code}") + code, _, _ = post_file(args.url, "/v2/events/..%2F..%2Fx/attachments/", args.key, jpg) + check("Attachment auf ungueltige object_id -> 404", code == 404, f"HTTP {code}") + code, _, _ = post_file(args.url, f"/v2/events/{status_id}/attachments/", args.key, jpg) + check("Attachment auf Status-Event -> 404", code == 404, f"HTTP {code}") + + if object_id: + code, headers, _ = post_file(args.url, f"/v2/events/{object_id}/attachments", args.key, jpg) + check("Attachment ohne Slash -> 308", code == 308, f"HTTP {code}") + code, _, data = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg) + check("Attachment", code == 200, f"HTTP {code} {data[:120]!r}") + code, _, _ = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg) + check("Attachment erneut (duplicate, 200)", code == 200, f"HTTP {code}") + + print(f"\nobject_id cameratrap_rep: {object_id}") + print(f"object_id status_report: {status_id}") + print(f"{failures} Fehler") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/migrate_legacy.py b/tools/migrate_legacy.py new file mode 100644 index 0000000..a867353 --- /dev/null +++ b/tools/migrate_legacy.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection). + + python -m tools.migrate_legacy # PG_DSN aus der Umgebung + python -m tools.migrate_legacy --dsn postgresql://... --batch 20000 + python -m tools.migrate_legacy --verify-only + +Wiederholbar. Jeder Batch ist eine eigene Transaktion. +""" +import argparse +import os +import re +import sys +import time +from pathlib import Path + +import psycopg + +SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema" +SCHEMA_LOCK_KEY = 7_311_000_001 + + +def load_steps() -> list[tuple[str, str]]: + sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8") + parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE) + return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)] + + +def apply_schema(conn: psycopg.Connection) -> None: + sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8") + with conn.transaction(), conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,)) + cur.execute(sql) + + +def migrate(conn: psycopg.Connection, batch_size: int) -> None: + steps = load_steps() + totals = {name: 0 for name, _ in steps} + with conn.cursor() as cur: + cur.execute("SELECT count(*) FROM remote_cam.metadata") + total_rows = cur.fetchone()[0] + cur.execute( + "CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS" + ) + conn.commit() + + last = "" + seen = 0 + t0 = time.perf_counter() + while True: + with conn.transaction(), conn.cursor() as cur: + cur.execute( + """ + INSERT INTO _mig_batch + SELECT metadata_uuid FROM remote_cam.metadata + WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s + """, + (last, batch_size), + ) + n = cur.rowcount + if n == 0: + break + cur.execute("SELECT max(metadata_uuid) FROM _mig_batch") + last = cur.fetchone()[0] + added = [] + for name, sql in steps: + cur.execute(sql) + totals[name] += cur.rowcount + added.append(f"{name}+{cur.rowcount}") + seen += n + rate = seen / max(time.perf_counter() - t0, 1e-6) + print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True) + + print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True) + + +def verify(conn: psycopg.Connection) -> bool: + checks = [ + ("metadata == image(entrance)", + "SELECT (SELECT count(*) FROM remote_cam.metadata), " + "(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"), + ("metadata == camera_status(entrance)", + "SELECT (SELECT count(*) FROM remote_cam.metadata), " + "(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"), + ("category == detection(entrance)", + "SELECT (SELECT count(*) FROM remote_cam.category), " + "(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"), + ("metadata ohne ocr_ts == image FAIL", + "SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), " + "(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"), + ("keine 0/0-Koordinaten (image, camera_status)", + "SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + " + "(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"), + ("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)", + """ + SELECT 0, count(*) FROM ( + SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200 + ) m + JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid + WHERE v.status_reported_at IS DISTINCT FROM m.servertime + OR v.battery_pct IS DISTINCT FROM m."BatteryStatus" + OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion" + OR v.status_camera_name IS DISTINCT FROM m."CameraName" + OR v.signal_strength IS DISTINCT FROM m."SignalStrength" + OR v.temperature_c IS DISTINCT FROM m."Temperature" + OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone" + OR v.ocr_ts IS DISTINCT FROM m.ocr_ts + """), + ] + ok_all = True + with conn.cursor() as cur: + for label, sql in checks: + cur.execute(sql) + expected, actual = cur.fetchone() + ok = expected == actual + ok_all &= ok + print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True) + conn.rollback() + return ok_all + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dsn", default=os.environ.get("PG_DSN")) + ap.add_argument("--batch", type=int, default=20000) + ap.add_argument("--verify-only", action="store_true") + args = ap.parse_args() + if not args.dsn: + print("--dsn oder PG_DSN fehlt", file=sys.stderr) + return 2 + + with psycopg.connect(args.dsn, connect_timeout=10) as conn: + if not args.verify_only: + apply_schema(conn) + print("Schema angewendet. Migriere ...", flush=True) + migrate(conn, args.batch) + print("Verifikation:", flush=True) + return 0 if verify(conn) else 1 + + +if __name__ == "__main__": + raise SystemExit(main())