Make the Gundi receiver robust against S3/DB outages

- Upload request path uses its own S3 client with short timeouts (5s/30s,
  2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB
  failures answer 503 + Retry-After instead of 500 after minutes of waiting.
- Transient S3/DB errors in ingest/OCR no longer count against max_attempts
  (previously ~15 min of outage turned images into FAILED); the sweep retries.
- The sweep no longer queues the same id twice (no pile-up during long outages).
- Backlog check logs [GUNDI][ALARM] for stuck/failed entries.
- tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-05 16:40:48 +02:00
co-authored by Claude Opus 5.5
parent cb2566a537
commit 6d518b4542
7 changed files with 659 additions and 21 deletions
+144 -14
View File
@@ -14,6 +14,7 @@ from typing import Optional
import psycopg
import yaml
from botocore.config import Config as BotoClientConfig
from dateutil import tz
from psycopg.types.json import Json
@@ -21,6 +22,7 @@ from main import (
AppConfig,
S3,
exif_write_with_exiftool,
is_transient_error,
make_thumbnail_bytes,
ocr_extract_timestamp,
to_exif_datetime,
@@ -55,6 +57,12 @@ class GundiConfig:
sweep_seconds: int = 60
stale_after_seconds: int = 300
max_attempts: int = 3
# S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen
upload_connect_timeout_seconds: int = 5
upload_read_timeout_seconds: int = 30
upload_max_attempts: int = 2 # inkl. erstem Versuch
# Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen
alarm_after_seconds: int = 1800
def load_gundi_config(path: str) -> GundiConfig:
@@ -82,6 +90,10 @@ def load_gundi_config(path: str) -> GundiConfig:
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))),
upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))),
upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))),
alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))),
)
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
raise SystemExit(
@@ -329,28 +341,38 @@ def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'."""
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.
Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung
blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro
object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt.
"""
with connect(dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE",
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s",
(object_id,),
)
row = cur.fetchone()
if not row or row[0] not in EVENT_CAMERATRAP_TYPES:
raise LookupError(object_id)
if row[1] in ("RECEIVED", "INGESTED"):
return "duplicate"
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
s3.put_bytes(key, jpg, "image/jpeg")
if not row or row[0] not in EVENT_CAMERATRAP_TYPES:
raise LookupError(object_id)
if row[1] in ("RECEIVED", "INGESTED"):
return "duplicate"
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
s3.put_bytes(key, jpg, "image/jpeg")
with connect(dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
UPDATE remote_cam.gundi_inbox
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
attempts = 0, last_error = NULL, updated_at = now()
WHERE object_id = %s
WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED')
""",
(key, object_id),
)
if cur.rowcount == 0:
return "duplicate"
return "stored"
@@ -384,10 +406,33 @@ class Pipeline:
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
self.cfg = cfg
self.gcfg = gcfg
self.s3 = S3(cfg)
# Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell.
# Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt).
self.s3 = S3(cfg, self._s3_config(
cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds,
cfg.s3_max_attempts, cfg.s3_retry_mode,
))
self.s3_request = S3(cfg, self._s3_config(
cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds,
gcfg.upload_max_attempts, "standard",
))
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
self._stop = threading.Event()
# Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf)
self._inflight: set[tuple[str, str]] = set()
self._inflight_lock = threading.Lock()
self._last_alarm = 0.0
@staticmethod
def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int,
max_attempts: int, retry_mode: str) -> BotoClientConfig:
return BotoClientConfig(
max_pool_connections=cfg.s3_max_pool_connections,
connect_timeout=connect_timeout,
read_timeout=read_timeout,
retries={"mode": retry_mode, "total_max_attempts": max_attempts},
)
def start(self) -> None:
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
@@ -398,17 +443,32 @@ class Pipeline:
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
def submit_ingest(self, object_id: str) -> None:
self._ingest_pool.submit(self._guarded, self.ingest, object_id)
self._submit(self._ingest_pool, self.ingest, object_id)
def submit_ocr(self, image_uuid: str) -> None:
self._ocr_pool.submit(self._guarded, self.ocr, image_uuid)
self._submit(self._ocr_pool, self.ocr, image_uuid)
@staticmethod
def _guarded(fn, arg: str) -> None:
def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None:
token = (fn.__name__, arg)
with self._inflight_lock:
if token in self._inflight:
return
self._inflight.add(token)
try:
pool.submit(self._guarded, token, fn, arg)
except Exception:
with self._inflight_lock:
self._inflight.discard(token)
raise
def _guarded(self, token: tuple[str, str], fn, arg: str) -> None:
try:
fn(arg)
except Exception as e:
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
finally:
with self._inflight_lock:
self._inflight.discard(token)
# --- Stufe 1 ---------------------------------------------------------
def ingest(self, object_id: str) -> None:
@@ -480,6 +540,20 @@ class Pipeline:
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
msg = f"{type(err).__name__}: {err}"[:1000]
if is_transient_error(err):
# S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min
# Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder.
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
UPDATE remote_cam.gundi_inbox
SET last_error = %s, updated_at = now()
WHERE object_id = %s AND state = 'RECEIVED'
""",
(msg, object_id),
)
print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
return
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
@@ -578,6 +652,18 @@ class Pipeline:
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
msg = f"{type(err).__name__}: {err}"[:1000]
if is_transient_error(err):
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
UPDATE remote_cam.image
SET ocr_error = %s, updated_at = now()
WHERE image_uuid = %s AND ocr_state = 'PENDING'
""",
(msg, image_uuid),
)
print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
return
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
@@ -645,3 +731,47 @@ class Pipeline:
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
flush=True,
)
self.check_backlog()
def check_backlog(self) -> None:
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
limit = self.gcfg.alarm_after_seconds
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT
count(*) FILTER (WHERE state = 'RECEIVED'
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
AND received_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'FAILED')
FROM remote_cam.gundi_inbox
WHERE received_at > now() - interval '14 days'
""",
(limit, limit),
)
received_stuck, awaiting_old, failed = cur.fetchone()
cur.execute(
"""
SELECT
count(*) FILTER (WHERE ocr_state = 'PENDING'
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE ocr_state = 'FAIL')
FROM remote_cam.image
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
""",
(limit,),
)
ocr_stuck, ocr_failed = cur.fetchone()
if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed):
return
now = time.monotonic()
if now - self._last_alarm < 300:
return
self._last_alarm = now
print(
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
"- bei failed/ocr_fail: tools/gundi_requeue.py",
flush=True,
)