Add Gundi v2 push receiver for ICU and normalized data model
ICU now pushes camera events via a Gundi-v2-compatible API instead of dropping file triplets into S3. The new gundi-receiver service accepts them directly, so images are in the system within seconds instead of waiting out the 90s age gate + poll interval. OCR of the burnt-in camera timestamp stays mandatory (ICU only sends server times; OCR detects drifting camera clocks) but now runs asynchronously after ingest. The push path writes a normalized model that separates what the legacy metadata table conflates: camera_status (status time series), image (image properties) and detection (classifications), plus the image_with_status view as the portal's migration target. A repeatable, batched migration splits existing metadata/category rows into it. The legacy poller is unchanged and keeps writing metadata/category until the portal is switched over. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,646 @@
|
||||
"""Push-Pfad (Gundi v2): Mapping ins Fachmodell, Ingest (Stufe 1), OCR (Stufe 2), Sweep."""
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid as uuidlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import psycopg
|
||||
import yaml
|
||||
from dateutil import tz
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from main import (
|
||||
AppConfig,
|
||||
S3,
|
||||
exif_write_with_exiftool,
|
||||
make_thumbnail_bytes,
|
||||
ocr_extract_timestamp,
|
||||
to_exif_datetime,
|
||||
)
|
||||
|
||||
SCHEMA_FILE = Path(__file__).resolve().parent / "schema" / "001_domain_model.sql"
|
||||
SCHEMA_LOCK_KEY = 7_311_000_001
|
||||
UUID_STRICT_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
|
||||
EVENT_CAMERATRAP = "cameratrap_rep"
|
||||
EVENT_STATUS = "camera_status_report"
|
||||
MIN_API_KEY_LEN = 24
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Config
|
||||
# -----------------------
|
||||
@dataclass
|
||||
class GundiConfig:
|
||||
enabled: bool
|
||||
api_key: str
|
||||
bind_host: str = "0.0.0.0"
|
||||
bind_port: int = 8090
|
||||
inbox_prefix: str = "icu/gundi-test/inbox/"
|
||||
processed_prefix: str = "icu/gundi-test/processed/"
|
||||
thumb_prefix: str = "icu/gundi-test/thumbnails/"
|
||||
workers: int = 4
|
||||
ocr_workers: int = 2
|
||||
max_request_bytes: int = 32 * 1024 * 1024
|
||||
abandon_after_hours: int = 24
|
||||
sweep_seconds: int = 60
|
||||
stale_after_seconds: int = 300
|
||||
max_attempts: int = 3
|
||||
|
||||
|
||||
def load_gundi_config(path: str) -> GundiConfig:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
raw = yaml.safe_load(f) or {}
|
||||
g = raw.get("gundi") or {}
|
||||
defaults = GundiConfig(enabled=False, api_key="")
|
||||
|
||||
def prefix(key: str) -> str:
|
||||
value = str(g.get(key, getattr(defaults, key)))
|
||||
return value if value.endswith("/") else value + "/"
|
||||
|
||||
gcfg = GundiConfig(
|
||||
enabled=bool(g.get("enabled", False)),
|
||||
api_key=os.environ.get("GUNDI_API_KEY", ""),
|
||||
bind_host=str(g.get("bind_host", defaults.bind_host)),
|
||||
bind_port=int(g.get("bind_port", defaults.bind_port)),
|
||||
inbox_prefix=prefix("inbox_prefix"),
|
||||
processed_prefix=prefix("processed_prefix"),
|
||||
thumb_prefix=prefix("thumb_prefix"),
|
||||
workers=max(1, int(g.get("workers", defaults.workers))),
|
||||
ocr_workers=max(1, int(g.get("ocr_workers", defaults.ocr_workers))),
|
||||
max_request_bytes=int(g.get("max_request_bytes", defaults.max_request_bytes)),
|
||||
abandon_after_hours=int(g.get("abandon_after_hours", defaults.abandon_after_hours)),
|
||||
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
|
||||
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
|
||||
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
|
||||
)
|
||||
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
|
||||
raise SystemExit(
|
||||
f"GUNDI_API_KEY fehlt oder ist kuerzer als {MIN_API_KEY_LEN} Zeichen "
|
||||
"(gundi.enabled ist aktiv). Bitte in der .env setzen."
|
||||
)
|
||||
return gcfg
|
||||
|
||||
|
||||
def connect(dsn: str) -> psycopg.Connection:
|
||||
return psycopg.connect(dsn, connect_timeout=10)
|
||||
|
||||
|
||||
def apply_schema(dsn: str) -> None:
|
||||
sql = SCHEMA_FILE.read_text(encoding="utf-8")
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
def utc_now_str() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Validierung + Mapping
|
||||
# -----------------------
|
||||
class EventError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _num(v) -> Optional[float]:
|
||||
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
||||
return None
|
||||
return float(v)
|
||||
|
||||
|
||||
def _int(v) -> Optional[int]:
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
if isinstance(v, int):
|
||||
return v
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str) and re.fullmatch(r"-?\d+", v.strip()):
|
||||
return int(v.strip())
|
||||
return None
|
||||
|
||||
|
||||
def _bool(v) -> Optional[bool]:
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if isinstance(v, str) and v.strip().lower() in ("true", "false"):
|
||||
return v.strip().lower() == "true"
|
||||
return None
|
||||
|
||||
|
||||
def _str(v) -> Optional[str]:
|
||||
if v is None:
|
||||
return None
|
||||
s = str(v).strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def parse_ts(value) -> datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise EventError("recorded_at fehlt")
|
||||
s = value.strip()
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
raise EventError(f"recorded_at ungueltig: {value!r}") from None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def coords(location) -> tuple[Optional[float], Optional[float]]:
|
||||
"""ICU meldet unbekannte Koordinaten als 0.0/0.0 -> NULL."""
|
||||
if not isinstance(location, dict):
|
||||
return None, None
|
||||
lat, lon = _num(location.get("lat")), _num(location.get("lon"))
|
||||
if lat is None or lon is None or (lat == 0 and lon == 0):
|
||||
return None, None
|
||||
return lat, lon
|
||||
|
||||
|
||||
def validate_event(ev) -> dict:
|
||||
if not isinstance(ev, dict):
|
||||
raise EventError("Event muss ein JSON-Objekt sein")
|
||||
event_type = ev.get("event_type")
|
||||
if not isinstance(event_type, str) or not event_type:
|
||||
raise EventError("event_type fehlt")
|
||||
source = ev.get("source")
|
||||
if not isinstance(source, str) or not source.strip():
|
||||
raise EventError("source fehlt")
|
||||
details = ev.get("event_details")
|
||||
if details is None:
|
||||
details = {}
|
||||
if not isinstance(details, dict):
|
||||
raise EventError("event_details muss ein Objekt sein")
|
||||
if event_type == EVENT_CAMERATRAP:
|
||||
dets = details.get("detections", [])
|
||||
if not isinstance(dets, list) or not all(isinstance(d, dict) for d in dets):
|
||||
raise EventError("event_details.detections muss eine Liste von Objekten sein")
|
||||
lat, lon = coords(ev.get("location"))
|
||||
return {
|
||||
"event_type": event_type,
|
||||
"source": source.strip(),
|
||||
"recorded_at": parse_ts(ev.get("recorded_at")),
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
|
||||
def camera_status_row(env: dict, object_id: str, raw: dict) -> dict:
|
||||
d = env["details"]
|
||||
return {
|
||||
"imei": env["source"],
|
||||
"reported_at": env["recorded_at"],
|
||||
"iccid": _str(d.get("iccid")),
|
||||
"init_key": _str(d.get("init_key")),
|
||||
"camera_name": _str(d.get("camera_name")),
|
||||
"camera_model": _str(d.get("camera_model")),
|
||||
"firmware_version": _str(d.get("firmware_version")),
|
||||
"camera_language": _str(d.get("camera_language")),
|
||||
"customer_timezone": _str(d.get("customer_timezone")),
|
||||
"mcc": _str(d.get("mobile_country_code")),
|
||||
"mnc": _str(d.get("mobile_network_code")),
|
||||
"lac": _str(d.get("location_area_code")),
|
||||
"cell_id": _str(d.get("cell_id")),
|
||||
"battery_pct": _int(d.get("battery_status")),
|
||||
"signal_strength": _int(d.get("signal_strength")),
|
||||
"temperature_c": _int(d.get("temperature")),
|
||||
"coordinate_switch": _bool(d.get("coordinate_switch")),
|
||||
"work_period": _bool(d.get("work_period")),
|
||||
"work_start": _str(d.get("work_start")),
|
||||
"work_end": _str(d.get("work_end")),
|
||||
"image_format": _str(d.get("image_format")),
|
||||
"image_night": _str(d.get("image_night")),
|
||||
"thumbnail_size": _str(d.get("thumbnail_size")),
|
||||
"pir_interval": _str(d.get("pir_interval")),
|
||||
"time_scan": _str(d.get("time_scan")),
|
||||
"lat": env["lat"],
|
||||
"lon": env["lon"],
|
||||
"source": "gundi",
|
||||
"source_ref": object_id,
|
||||
"raw": Json(raw),
|
||||
}
|
||||
|
||||
|
||||
def detection_rows(env: dict, image_uuid: str) -> list[dict]:
|
||||
rows = []
|
||||
for det in env["details"].get("detections") or []:
|
||||
category = _str(det.get("category"))
|
||||
if category is None:
|
||||
continue
|
||||
pct = _num(det.get("confidence_percent"))
|
||||
confidence = None if pct is None else min(max(pct / 100.0, 0.0), 1.0)
|
||||
bbox = [_int(det.get(k)) for k in ("x_px", "y_px", "width_px", "height_px")]
|
||||
if all(v in (None, 0) for v in bbox):
|
||||
bbox = [None, None, None, None]
|
||||
rows.append({
|
||||
"image_uuid": image_uuid,
|
||||
"category": category,
|
||||
"confidence": confidence,
|
||||
"bbox_x": bbox[0],
|
||||
"bbox_y": bbox[1],
|
||||
"bbox_w": bbox[2],
|
||||
"bbox_h": bbox[3],
|
||||
"model_version": "icu",
|
||||
"source": "gundi",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def image_row(env: dict, object_id: str, raw: dict, size_bytes: int, processed_key: str, thumb_key: str) -> dict:
|
||||
d = env["details"]
|
||||
return {
|
||||
"image_uuid": object_id,
|
||||
"imei": env["source"],
|
||||
"servertime": env["recorded_at"],
|
||||
"ocr_state": "PENDING",
|
||||
"camera_name": _str(d.get("cameratraprep_camera-name")),
|
||||
"camera_make": _str(d.get("cameratraprep_camera-make")),
|
||||
"lat": env["lat"],
|
||||
"lon": env["lon"],
|
||||
"size_bytes": size_bytes,
|
||||
"processed_key": processed_key,
|
||||
"thumbnail_key": thumb_key,
|
||||
"source": "gundi",
|
||||
"source_ref": object_id,
|
||||
"raw": Json(raw),
|
||||
}
|
||||
|
||||
|
||||
def _insert(cur, table: str, row: dict, suffix: str = "") -> int:
|
||||
cols = list(row)
|
||||
cur.execute(
|
||||
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) {suffix}",
|
||||
[row[c] for c in cols],
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Receiver-seitige Persistenz (synchron im Request)
|
||||
# -----------------------
|
||||
def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
|
||||
"""Ein Request = eine Transaktion. Antwort erst nach Commit."""
|
||||
results = []
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
for env, raw in items:
|
||||
object_id = str(uuidlib.uuid4())
|
||||
if env["event_type"] == EVENT_STATUS:
|
||||
state = "INGESTED"
|
||||
elif env["event_type"] == EVENT_CAMERATRAP:
|
||||
state = "AWAITING_ATTACHMENT"
|
||||
else:
|
||||
state = "IGNORED"
|
||||
_insert(cur, "remote_cam.gundi_inbox", {
|
||||
"object_id": object_id,
|
||||
"event_type": env["event_type"],
|
||||
"source": env["source"],
|
||||
"recorded_at": env["recorded_at"],
|
||||
"payload": Json(raw),
|
||||
"state": state,
|
||||
})
|
||||
if env["event_type"] == EVENT_STATUS:
|
||||
_insert(
|
||||
cur, "remote_cam.camera_status", camera_status_row(env, object_id, raw),
|
||||
"ON CONFLICT (imei, reported_at) WHERE source = 'gundi' DO NOTHING",
|
||||
)
|
||||
results.append({
|
||||
"object_id": object_id,
|
||||
"created_at": utc_now_str(),
|
||||
"event_type": env["event_type"],
|
||||
"source": env["source"],
|
||||
"state": state,
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
|
||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'."""
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE",
|
||||
(object_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row or row[0] != EVENT_CAMERATRAP:
|
||||
raise LookupError(object_id)
|
||||
if row[1] in ("RECEIVED", "INGESTED"):
|
||||
return "duplicate"
|
||||
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
||||
s3.put_bytes(key, jpg, "image/jpeg")
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
|
||||
attempts = 0, last_error = NULL, updated_at = now()
|
||||
WHERE object_id = %s
|
||||
""",
|
||||
(key, object_id),
|
||||
)
|
||||
return "stored"
|
||||
|
||||
|
||||
# -----------------------
|
||||
# EXIF
|
||||
# -----------------------
|
||||
def exif_set_digitized(jpg: bytes, servertime: datetime, *, timeout_seconds: int) -> bytes:
|
||||
"""Stufe 1: nur Serverzeit. DateTimeOriginal folgt nach dem OCR (Stufe 2)."""
|
||||
dtd = to_exif_datetime(servertime.astimezone(timezone.utc))
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
path = os.path.join(td, "in.jpg")
|
||||
with open(path, "wb") as f:
|
||||
f.write(jpg)
|
||||
subprocess.run(
|
||||
[
|
||||
"exiftool", "-overwrite_original",
|
||||
f"-DateTimeDigitized={dtd}", "-OffsetTimeDigitized=+00:00",
|
||||
f"-ModifyDate={dtd}", "-OffsetTime=+00:00",
|
||||
path,
|
||||
],
|
||||
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_seconds,
|
||||
)
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Pipeline
|
||||
# -----------------------
|
||||
class Pipeline:
|
||||
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
|
||||
self.cfg = cfg
|
||||
self.gcfg = gcfg
|
||||
self.s3 = S3(cfg)
|
||||
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
|
||||
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self) -> None:
|
||||
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._ingest_pool.shutdown(wait=False, cancel_futures=True)
|
||||
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_ingest(self, object_id: str) -> None:
|
||||
self._ingest_pool.submit(self._guarded, self.ingest, object_id)
|
||||
|
||||
def submit_ocr(self, image_uuid: str) -> None:
|
||||
self._ocr_pool.submit(self._guarded, self.ocr, image_uuid)
|
||||
|
||||
@staticmethod
|
||||
def _guarded(fn, arg: str) -> None:
|
||||
try:
|
||||
fn(arg)
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
|
||||
|
||||
# --- Stufe 1 ---------------------------------------------------------
|
||||
def ingest(self, object_id: str) -> None:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT payload, attachment_key FROM remote_cam.gundi_inbox
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
(object_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return
|
||||
payload, attachment_key = row
|
||||
env = validate_event(payload)
|
||||
jpg = self.s3.get_bytes(attachment_key)
|
||||
|
||||
exif_note = "ok"
|
||||
try:
|
||||
processed = exif_set_digitized(
|
||||
jpg, env["recorded_at"], timeout_seconds=self.cfg.exiftool_timeout_seconds
|
||||
)
|
||||
except Exception as e:
|
||||
processed = jpg
|
||||
exif_note = f"fail({type(e).__name__})"
|
||||
thumb = make_thumbnail_bytes(jpg, self.cfg.thumb_max)
|
||||
|
||||
processed_key = f"{self.gcfg.processed_prefix}{object_id}.jpg"
|
||||
thumb_key = f"{self.gcfg.thumb_prefix}{object_id}.jpg"
|
||||
self.s3.put_bytes(processed_key, processed, "image/jpeg")
|
||||
self.s3.put_bytes(thumb_key, thumb, "image/jpeg")
|
||||
|
||||
n_det = 0
|
||||
inserted = _insert(
|
||||
cur, "remote_cam.image",
|
||||
image_row(env, object_id, payload, len(jpg), processed_key, thumb_key),
|
||||
"ON CONFLICT (image_uuid) DO NOTHING",
|
||||
)
|
||||
if inserted:
|
||||
for det in detection_rows(env, object_id):
|
||||
n_det += _insert(cur, "remote_cam.detection", det)
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'INGESTED', last_error = NULL, updated_at = now()
|
||||
WHERE object_id = %s
|
||||
""",
|
||||
(object_id,),
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_ingest_failure(object_id, e)
|
||||
return
|
||||
|
||||
try:
|
||||
self.s3.delete_keys([attachment_key])
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] inbox-cleanup {object_id} fehlgeschlagen: {e}", flush=True)
|
||||
self.submit_ocr(object_id)
|
||||
ms = int((time.perf_counter() - start) * 1000)
|
||||
print(
|
||||
f"[GUNDI] ingest object_id={object_id} imei={env['source']} detections={n_det} "
|
||||
f"exif={exif_note} ms={ms}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET attempts = attempts + 1,
|
||||
state = CASE WHEN attempts + 1 >= %s THEN 'FAILED' ELSE state END,
|
||||
last_error = %s, updated_at = now()
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
RETURNING attempts, state
|
||||
""",
|
||||
(self.gcfg.max_attempts, msg, object_id),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
print(f"[GUNDI] ingest object_id={object_id} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True)
|
||||
|
||||
# --- Stufe 2 ---------------------------------------------------------
|
||||
def ocr(self, image_uuid: str) -> None:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT imei, servertime, processed_key FROM remote_cam.image
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
(image_uuid,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return
|
||||
imei, servertime, processed_key = row
|
||||
tzinfo, tzname = self._camera_tz(cur, imei, servertime)
|
||||
jpg = self.s3.get_bytes(processed_key)
|
||||
ocr_ts = ocr_extract_timestamp(
|
||||
jpg, self.cfg.ocr_crop_w_frac, self.cfg.ocr_crop_h_frac, tzinfo
|
||||
)
|
||||
|
||||
exif_err = None
|
||||
try:
|
||||
patched = exif_write_with_exiftool(
|
||||
jpg, ocr_ts, servertime, timeout_seconds=self.cfg.exiftool_timeout_seconds
|
||||
)
|
||||
self.s3.put_bytes(processed_key, patched, "image/jpeg")
|
||||
except Exception as e:
|
||||
exif_err = f"exif: {type(e).__name__}: {e}"[:1000]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_ts = %s, ocr_state = 'OK', ocr_attempts = ocr_attempts + 1,
|
||||
ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s
|
||||
""",
|
||||
(ocr_ts, exif_err, image_uuid),
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_ocr_failure(image_uuid, e)
|
||||
return
|
||||
|
||||
drift = int((servertime - ocr_ts).total_seconds())
|
||||
ms = int((time.perf_counter() - start) * 1000)
|
||||
print(
|
||||
f"[GUNDI] ocr image_uuid={image_uuid} ocr_ts={ocr_ts.isoformat()} tz={tzname} "
|
||||
f"servertime-ocr={drift}s exif={'ok' if exif_err is None else 'fail'} ms={ms}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _camera_tz(cur, imei: str, servertime: datetime):
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT customer_timezone FROM remote_cam.camera_status
|
||||
WHERE imei = %s AND customer_timezone IS NOT NULL AND reported_at <= %s
|
||||
ORDER BY reported_at DESC LIMIT 1
|
||||
""",
|
||||
(imei, servertime),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT customer_timezone FROM remote_cam.camera_status
|
||||
WHERE imei = %s AND customer_timezone IS NOT NULL
|
||||
ORDER BY reported_at ASC LIMIT 1
|
||||
""",
|
||||
(imei,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
name = row[0] if row else "UTC"
|
||||
zone = tz.gettz(name)
|
||||
if zone is None:
|
||||
return timezone.utc, "UTC"
|
||||
return zone, name
|
||||
|
||||
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_attempts = ocr_attempts + 1,
|
||||
ocr_state = CASE WHEN ocr_attempts + 1 >= %s THEN 'FAIL' ELSE ocr_state END,
|
||||
ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
RETURNING ocr_attempts, ocr_state
|
||||
""",
|
||||
(self.gcfg.max_attempts, msg, image_uuid),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
print(f"[GUNDI] ocr image_uuid={image_uuid} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True)
|
||||
|
||||
# --- Sweep -----------------------------------------------------------
|
||||
def _sweep_loop(self) -> None:
|
||||
delay = 0
|
||||
while not self._stop.wait(delay):
|
||||
delay = self.gcfg.sweep_seconds
|
||||
try:
|
||||
self.sweep()
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] sweep Fehler: {type(e).__name__}: {e}", flush=True)
|
||||
|
||||
def sweep(self) -> None:
|
||||
stale = self.gcfg.stale_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT object_id::text FROM remote_cam.gundi_inbox
|
||||
WHERE state = 'RECEIVED' AND updated_at < now() - make_interval(secs => %s)
|
||||
ORDER BY updated_at LIMIT 200
|
||||
""",
|
||||
(stale,),
|
||||
)
|
||||
ingest_ids = [r[0] for r in cur.fetchall()]
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT image_uuid::text FROM remote_cam.image
|
||||
WHERE ocr_state = 'PENDING' AND source = 'gundi'
|
||||
AND updated_at < now() - make_interval(secs => %s)
|
||||
ORDER BY updated_at LIMIT 200
|
||||
""",
|
||||
(stale,),
|
||||
)
|
||||
ocr_ids = [r[0] for r in cur.fetchall()]
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'ABANDONED', updated_at = now()
|
||||
WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(hours => %s)
|
||||
""",
|
||||
(self.gcfg.abandon_after_hours,),
|
||||
)
|
||||
abandoned = cur.rowcount
|
||||
for object_id in ingest_ids:
|
||||
self.submit_ingest(object_id)
|
||||
for image_uuid in ocr_ids:
|
||||
self.submit_ocr(image_uuid)
|
||||
if ingest_ids or ocr_ids or abandoned:
|
||||
print(
|
||||
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
|
||||
flush=True,
|
||||
)
|
||||
Reference in New Issue
Block a user