Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+144
-14
@@ -14,6 +14,7 @@ from typing import Optional
|
||||
|
||||
import psycopg
|
||||
import yaml
|
||||
from botocore.config import Config as BotoClientConfig
|
||||
from dateutil import tz
|
||||
from psycopg.types.json import Json
|
||||
|
||||
@@ -21,6 +22,7 @@ from main import (
|
||||
AppConfig,
|
||||
S3,
|
||||
exif_write_with_exiftool,
|
||||
is_transient_error,
|
||||
make_thumbnail_bytes,
|
||||
ocr_extract_timestamp,
|
||||
to_exif_datetime,
|
||||
@@ -55,6 +57,12 @@ class GundiConfig:
|
||||
sweep_seconds: int = 60
|
||||
stale_after_seconds: int = 300
|
||||
max_attempts: int = 3
|
||||
# S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen
|
||||
upload_connect_timeout_seconds: int = 5
|
||||
upload_read_timeout_seconds: int = 30
|
||||
upload_max_attempts: int = 2 # inkl. erstem Versuch
|
||||
# Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen
|
||||
alarm_after_seconds: int = 1800
|
||||
|
||||
|
||||
def load_gundi_config(path: str) -> GundiConfig:
|
||||
@@ -82,6 +90,10 @@ def load_gundi_config(path: str) -> GundiConfig:
|
||||
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
|
||||
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
|
||||
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
|
||||
upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))),
|
||||
upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))),
|
||||
upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))),
|
||||
alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))),
|
||||
)
|
||||
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
|
||||
raise SystemExit(
|
||||
@@ -329,28 +341,38 @@ def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
|
||||
|
||||
|
||||
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
|
||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'."""
|
||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.
|
||||
|
||||
Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung
|
||||
blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro
|
||||
object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt.
|
||||
"""
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE",
|
||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s",
|
||||
(object_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row or row[0] not in EVENT_CAMERATRAP_TYPES:
|
||||
raise LookupError(object_id)
|
||||
if row[1] in ("RECEIVED", "INGESTED"):
|
||||
return "duplicate"
|
||||
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
||||
s3.put_bytes(key, jpg, "image/jpeg")
|
||||
if not row or row[0] not in EVENT_CAMERATRAP_TYPES:
|
||||
raise LookupError(object_id)
|
||||
if row[1] in ("RECEIVED", "INGESTED"):
|
||||
return "duplicate"
|
||||
|
||||
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
||||
s3.put_bytes(key, jpg, "image/jpeg")
|
||||
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
|
||||
attempts = 0, last_error = NULL, updated_at = now()
|
||||
WHERE object_id = %s
|
||||
WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED')
|
||||
""",
|
||||
(key, object_id),
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
return "duplicate"
|
||||
return "stored"
|
||||
|
||||
|
||||
@@ -384,10 +406,33 @@ class Pipeline:
|
||||
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
|
||||
self.cfg = cfg
|
||||
self.gcfg = gcfg
|
||||
self.s3 = S3(cfg)
|
||||
# Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell.
|
||||
# Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt).
|
||||
self.s3 = S3(cfg, self._s3_config(
|
||||
cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds,
|
||||
cfg.s3_max_attempts, cfg.s3_retry_mode,
|
||||
))
|
||||
self.s3_request = S3(cfg, self._s3_config(
|
||||
cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds,
|
||||
gcfg.upload_max_attempts, "standard",
|
||||
))
|
||||
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
|
||||
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
|
||||
self._stop = threading.Event()
|
||||
# Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf)
|
||||
self._inflight: set[tuple[str, str]] = set()
|
||||
self._inflight_lock = threading.Lock()
|
||||
self._last_alarm = 0.0
|
||||
|
||||
@staticmethod
|
||||
def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int,
|
||||
max_attempts: int, retry_mode: str) -> BotoClientConfig:
|
||||
return BotoClientConfig(
|
||||
max_pool_connections=cfg.s3_max_pool_connections,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"mode": retry_mode, "total_max_attempts": max_attempts},
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
|
||||
@@ -398,17 +443,32 @@ class Pipeline:
|
||||
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_ingest(self, object_id: str) -> None:
|
||||
self._ingest_pool.submit(self._guarded, self.ingest, object_id)
|
||||
self._submit(self._ingest_pool, self.ingest, object_id)
|
||||
|
||||
def submit_ocr(self, image_uuid: str) -> None:
|
||||
self._ocr_pool.submit(self._guarded, self.ocr, image_uuid)
|
||||
self._submit(self._ocr_pool, self.ocr, image_uuid)
|
||||
|
||||
@staticmethod
|
||||
def _guarded(fn, arg: str) -> None:
|
||||
def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None:
|
||||
token = (fn.__name__, arg)
|
||||
with self._inflight_lock:
|
||||
if token in self._inflight:
|
||||
return
|
||||
self._inflight.add(token)
|
||||
try:
|
||||
pool.submit(self._guarded, token, fn, arg)
|
||||
except Exception:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(token)
|
||||
raise
|
||||
|
||||
def _guarded(self, token: tuple[str, str], fn, arg: str) -> None:
|
||||
try:
|
||||
fn(arg)
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(token)
|
||||
|
||||
# --- Stufe 1 ---------------------------------------------------------
|
||||
def ingest(self, object_id: str) -> None:
|
||||
@@ -480,6 +540,20 @@ class Pipeline:
|
||||
|
||||
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
if is_transient_error(err):
|
||||
# S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min
|
||||
# Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder.
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET last_error = %s, updated_at = now()
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
""",
|
||||
(msg, object_id),
|
||||
)
|
||||
print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||
return
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
@@ -578,6 +652,18 @@ class Pipeline:
|
||||
|
||||
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
if is_transient_error(err):
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
""",
|
||||
(msg, image_uuid),
|
||||
)
|
||||
print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||
return
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
@@ -645,3 +731,47 @@ class Pipeline:
|
||||
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
|
||||
flush=True,
|
||||
)
|
||||
self.check_backlog()
|
||||
|
||||
def check_backlog(self) -> None:
|
||||
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
|
||||
limit = self.gcfg.alarm_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE state = 'RECEIVED'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'FAILED')
|
||||
FROM remote_cam.gundi_inbox
|
||||
WHERE received_at > now() - interval '14 days'
|
||||
""",
|
||||
(limit, limit),
|
||||
)
|
||||
received_stuck, awaiting_old, failed = cur.fetchone()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE ocr_state = 'PENDING'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL')
|
||||
FROM remote_cam.image
|
||||
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
ocr_stuck, ocr_failed = cur.fetchone()
|
||||
if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed):
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now - self._last_alarm < 300:
|
||||
return
|
||||
self._last_alarm = now
|
||||
print(
|
||||
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
|
||||
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
|
||||
"- bei failed/ocr_fail: tools/gundi_requeue.py",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user