Make the Gundi receiver robust against S3/DB outages

- Upload request path uses its own S3 client with short timeouts (5s/30s,
  2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB
  failures answer 503 + Retry-After instead of 500 after minutes of waiting.
- Transient S3/DB errors in ingest/OCR no longer count against max_attempts
  (previously ~15 min of outage turned images into FAILED); the sweep retries.
- The sweep no longer queues the same id twice (no pile-up during long outages).
- Backlog check logs [GUNDI][ALARM] for stuck/failed entries.
- tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-05 16:40:48 +02:00
co-authored by Claude Opus 5.5
parent cb2566a537
commit 6d518b4542
7 changed files with 659 additions and 21 deletions
+18
View File
@@ -91,6 +91,24 @@ liest danach asynchron per OCR den Kamera-Zeitstempel, setzt `image.ocr_ts` und
`DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert `DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert
Events ohne Bild nach `abandon_after_hours` als `ABANDONED`. Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
### Verhalten bei S3-/DB-Ausfall
| Teil | Verhalten |
|------|-----------|
| Status-Events, Event anlegen | brauchen kein S3; bei DB-Ausfall `503` + `Retry-After: 30` |
| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen |
| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` |
| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) |
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren |
Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`):
```bash
docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue --apply
```
Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne Wiederholung geht das Bild verloren, wenn S3 laenger ausfaellt.
### Datenmodell (`schema/001_domain_model.sql`) ### Datenmodell (`schema/001_domain_model.sql`)
| Tabelle | Inhalt | | Tabelle | Inhalt |
+6
View File
@@ -65,3 +65,9 @@ gundi:
sweep_seconds: 60 sweep_seconds: 60
stale_after_seconds: 300 stale_after_seconds: 300
max_attempts: 3 max_attempts: 3
# S3-Ausfall-Robustheit (siehe README): Upload-Request scheitert schnell mit 503 + Retry-After;
# S3-/DB-Stoerungen zaehlen nicht gegen max_attempts (nur Datenfehler).
upload_connect_timeout_seconds: 5
upload_read_timeout_seconds: 30
upload_max_attempts: 2 # inkl. erstem Versuch
alarm_after_seconds: 1800 # [GUNDI][ALARM], wenn Eintraege laenger haengen
+138 -8
View File
@@ -14,6 +14,7 @@ from typing import Optional
import psycopg import psycopg
import yaml import yaml
from botocore.config import Config as BotoClientConfig
from dateutil import tz from dateutil import tz
from psycopg.types.json import Json from psycopg.types.json import Json
@@ -21,6 +22,7 @@ from main import (
AppConfig, AppConfig,
S3, S3,
exif_write_with_exiftool, exif_write_with_exiftool,
is_transient_error,
make_thumbnail_bytes, make_thumbnail_bytes,
ocr_extract_timestamp, ocr_extract_timestamp,
to_exif_datetime, to_exif_datetime,
@@ -55,6 +57,12 @@ class GundiConfig:
sweep_seconds: int = 60 sweep_seconds: int = 60
stale_after_seconds: int = 300 stale_after_seconds: int = 300
max_attempts: int = 3 max_attempts: int = 3
# S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen
upload_connect_timeout_seconds: int = 5
upload_read_timeout_seconds: int = 30
upload_max_attempts: int = 2 # inkl. erstem Versuch
# Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen
alarm_after_seconds: int = 1800
def load_gundi_config(path: str) -> GundiConfig: def load_gundi_config(path: str) -> GundiConfig:
@@ -82,6 +90,10 @@ def load_gundi_config(path: str) -> GundiConfig:
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)), sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)), stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))), max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))),
upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))),
upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))),
alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))),
) )
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN: if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
raise SystemExit( raise SystemExit(
@@ -329,10 +341,15 @@ def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str: def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.""" """Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.
Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung
blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro
object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt.
"""
with connect(dsn) as conn, conn.cursor() as cur: with connect(dsn) as conn, conn.cursor() as cur:
cur.execute( cur.execute(
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE", "SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s",
(object_id,), (object_id,),
) )
row = cur.fetchone() row = cur.fetchone()
@@ -340,17 +357,22 @@ def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes
raise LookupError(object_id) raise LookupError(object_id)
if row[1] in ("RECEIVED", "INGESTED"): if row[1] in ("RECEIVED", "INGESTED"):
return "duplicate" return "duplicate"
key = f"{gcfg.inbox_prefix}{object_id}.jpg" key = f"{gcfg.inbox_prefix}{object_id}.jpg"
s3.put_bytes(key, jpg, "image/jpeg") s3.put_bytes(key, jpg, "image/jpeg")
with connect(dsn) as conn, conn.cursor() as cur:
cur.execute( cur.execute(
""" """
UPDATE remote_cam.gundi_inbox UPDATE remote_cam.gundi_inbox
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(), SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
attempts = 0, last_error = NULL, updated_at = now() attempts = 0, last_error = NULL, updated_at = now()
WHERE object_id = %s WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED')
""", """,
(key, object_id), (key, object_id),
) )
if cur.rowcount == 0:
return "duplicate"
return "stored" return "stored"
@@ -384,10 +406,33 @@ class Pipeline:
def __init__(self, cfg: AppConfig, gcfg: GundiConfig): def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
self.cfg = cfg self.cfg = cfg
self.gcfg = gcfg self.gcfg = gcfg
self.s3 = S3(cfg) # Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell.
# Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt).
self.s3 = S3(cfg, self._s3_config(
cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds,
cfg.s3_max_attempts, cfg.s3_retry_mode,
))
self.s3_request = S3(cfg, self._s3_config(
cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds,
gcfg.upload_max_attempts, "standard",
))
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest") self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr") self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
self._stop = threading.Event() self._stop = threading.Event()
# Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf)
self._inflight: set[tuple[str, str]] = set()
self._inflight_lock = threading.Lock()
self._last_alarm = 0.0
@staticmethod
def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int,
max_attempts: int, retry_mode: str) -> BotoClientConfig:
return BotoClientConfig(
max_pool_connections=cfg.s3_max_pool_connections,
connect_timeout=connect_timeout,
read_timeout=read_timeout,
retries={"mode": retry_mode, "total_max_attempts": max_attempts},
)
def start(self) -> None: def start(self) -> None:
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start() threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
@@ -398,17 +443,32 @@ class Pipeline:
self._ocr_pool.shutdown(wait=False, cancel_futures=True) self._ocr_pool.shutdown(wait=False, cancel_futures=True)
def submit_ingest(self, object_id: str) -> None: def submit_ingest(self, object_id: str) -> None:
self._ingest_pool.submit(self._guarded, self.ingest, object_id) self._submit(self._ingest_pool, self.ingest, object_id)
def submit_ocr(self, image_uuid: str) -> None: def submit_ocr(self, image_uuid: str) -> None:
self._ocr_pool.submit(self._guarded, self.ocr, image_uuid) self._submit(self._ocr_pool, self.ocr, image_uuid)
@staticmethod def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None:
def _guarded(fn, arg: str) -> None: token = (fn.__name__, arg)
with self._inflight_lock:
if token in self._inflight:
return
self._inflight.add(token)
try:
pool.submit(self._guarded, token, fn, arg)
except Exception:
with self._inflight_lock:
self._inflight.discard(token)
raise
def _guarded(self, token: tuple[str, str], fn, arg: str) -> None:
try: try:
fn(arg) fn(arg)
except Exception as e: except Exception as e:
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True) print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
finally:
with self._inflight_lock:
self._inflight.discard(token)
# --- Stufe 1 --------------------------------------------------------- # --- Stufe 1 ---------------------------------------------------------
def ingest(self, object_id: str) -> None: def ingest(self, object_id: str) -> None:
@@ -480,6 +540,20 @@ class Pipeline:
def _record_ingest_failure(self, object_id: str, err: Exception) -> None: def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
msg = f"{type(err).__name__}: {err}"[:1000] msg = f"{type(err).__name__}: {err}"[:1000]
if is_transient_error(err):
# S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min
# Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder.
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
UPDATE remote_cam.gundi_inbox
SET last_error = %s, updated_at = now()
WHERE object_id = %s AND state = 'RECEIVED'
""",
(msg, object_id),
)
print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
return
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute( cur.execute(
""" """
@@ -578,6 +652,18 @@ class Pipeline:
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None: def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
msg = f"{type(err).__name__}: {err}"[:1000] msg = f"{type(err).__name__}: {err}"[:1000]
if is_transient_error(err):
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
UPDATE remote_cam.image
SET ocr_error = %s, updated_at = now()
WHERE image_uuid = %s AND ocr_state = 'PENDING'
""",
(msg, image_uuid),
)
print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
return
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute( cur.execute(
""" """
@@ -645,3 +731,47 @@ class Pipeline:
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}", f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
flush=True, flush=True,
) )
self.check_backlog()
def check_backlog(self) -> None:
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
limit = self.gcfg.alarm_after_seconds
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT
count(*) FILTER (WHERE state = 'RECEIVED'
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
AND received_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'FAILED')
FROM remote_cam.gundi_inbox
WHERE received_at > now() - interval '14 days'
""",
(limit, limit),
)
received_stuck, awaiting_old, failed = cur.fetchone()
cur.execute(
"""
SELECT
count(*) FILTER (WHERE ocr_state = 'PENDING'
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE ocr_state = 'FAIL')
FROM remote_cam.image
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
""",
(limit,),
)
ocr_stuck, ocr_failed = cur.fetchone()
if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed):
return
now = time.monotonic()
if now - self._last_alarm < 300:
return
self._last_alarm = now
print(
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
"- bei failed/ocr_fail: tools/gundi_requeue.py",
flush=True,
)
+13 -5
View File
@@ -29,9 +29,10 @@ from gundi_pipeline import (
utc_now_str, utc_now_str,
validate_event, validate_event,
) )
from main import AppConfig, load_config from main import AppConfig, is_transient_error, load_config
EVENTS_PATH = "/v2/events/" EVENTS_PATH = "/v2/events/"
RETRY_AFTER_SECONDS = 30
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$") ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
JPEG_MAGIC = b"\xff\xd8\xff" JPEG_MAGIC = b"\xff\xd8\xff"
@@ -73,18 +74,20 @@ class GundiHandler(BaseHTTPRequestHandler):
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True) print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
# --- Antworten ------------------------------------------------------- # --- Antworten -------------------------------------------------------
def _json(self, code: int, data) -> None: def _json(self, code: int, data, *, retry_after: int | None = None) -> None:
body = json.dumps(data, ensure_ascii=False).encode("utf-8") body = json.dumps(data, ensure_ascii=False).encode("utf-8")
self.send_response(code) self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Type", "application/json; charset=utf-8")
if retry_after is not None:
self.send_header("Retry-After", str(retry_after))
self.send_header("Content-Length", str(len(body))) self.send_header("Content-Length", str(len(body)))
self.end_headers() self.end_headers()
self.wfile.write(body) self.wfile.write(body)
def _error(self, code: int, message: str, *, close: bool = False) -> None: def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None:
if close: if close:
self.close_connection = True self.close_connection = True
self._json(code, {"error": message}) self._json(code, {"error": message}, retry_after=retry_after)
def _redirect(self, location: str) -> None: def _redirect(self, location: str) -> None:
# 308 statt 301: der Client muss POST + Body beibehalten. # 308 statt 301: der Client muss POST + Body beibehalten.
@@ -155,6 +158,11 @@ class GundiHandler(BaseHTTPRequestHandler):
else: else:
self._handle_events(body) self._handle_events(body)
except Exception as e: except Exception as e:
if is_transient_error(e):
# S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt
print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True)
self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS)
else:
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True) print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
self._error(500, "internal error", close=True) self._error(500, "internal error", close=True)
@@ -202,7 +210,7 @@ class GundiHandler(BaseHTTPRequestHandler):
return return
try: try:
outcome = attach_image( outcome = attach_image(
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg
) )
except LookupError: except LookupError:
self._error(404, "Unknown object_id") self._error(404, "Unknown object_id")
+252
View File
@@ -0,0 +1,252 @@
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
python -m unittest discover -s tests -v (oder: python -m pytest tests)
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
"""
import http.client
import json
import os
import sys
import threading
import unittest
import uuid
from types import SimpleNamespace
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import psycopg # noqa: E402
from botocore.exceptions import ClientError # noqa: E402
import gundi_pipeline as gp # noqa: E402
import gundi_receiver as gr # noqa: E402
def s3_unavailable() -> ClientError:
return ClientError(
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
"PutObject",
)
class FakeCursor:
def __init__(self, rows=None, rowcount=1):
self.executed: list[tuple[str, tuple]] = []
self._rows = list(rows or [])
self.rowcount = rowcount
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def execute(self, sql, params=()):
self.executed.append((" ".join(sql.split()), params))
def fetchone(self):
return self._rows.pop(0) if self._rows else None
class FakeConn:
def __init__(self, cursor: FakeCursor):
self._cursor = cursor
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def cursor(self):
return self._cursor
def fake_connect(cursor: FakeCursor):
return lambda dsn: FakeConn(cursor)
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
class AttachImageTests(unittest.TestCase):
def test_s3_down_raises_without_db_lock_and_without_update(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
s3 = mock.Mock()
s3.put_bytes.side_effect = s3_unavailable()
with mock.patch.object(gp, "connect", fake_connect(cur)):
with self.assertRaises(ClientError):
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
sqls = [sql for sql, _ in cur.executed]
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
def test_success_stores_then_updates(self):
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
def test_already_received_is_duplicate_without_s3(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
s3.put_bytes.assert_not_called()
def test_parallel_upload_lost_the_race_is_duplicate(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
def test_unknown_or_wrong_type_is_lookup_error(self):
for rows in ([], [("camera_status_report", "INGESTED")]):
cur = FakeCursor(rows=rows)
with mock.patch.object(gp, "connect", fake_connect(cur)):
with self.assertRaises(LookupError):
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
def bare_pipeline() -> gp.Pipeline:
p = gp.Pipeline.__new__(gp.Pipeline)
p.cfg = SimpleNamespace(pg_dsn="dsn")
p.gcfg = GCFG
p._inflight = set()
p._inflight_lock = threading.Lock()
return p
class FailureRecordingTests(unittest.TestCase):
def run_failure(self, method: str, err: Exception) -> str:
cur = FakeCursor(rows=[(1, "RECEIVED")])
p = bare_pipeline()
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
getattr(p, method)("id-1", err)
self.assertEqual(len(cur.executed), 1)
return cur.executed[0][0]
def test_ingest_transient_does_not_count_attempt(self):
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
self.assertNotIn("attempts + 1", sql)
self.assertNotIn("FAILED", sql)
def test_ingest_db_outage_does_not_count_attempt(self):
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
self.assertNotIn("attempts + 1", sql)
def test_ingest_permanent_counts_attempt(self):
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
self.assertIn("attempts + 1", sql)
self.assertIn("'FAILED'", sql)
def test_ocr_transient_does_not_count_attempt(self):
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
self.assertNotIn("ocr_attempts + 1", sql)
self.assertNotIn("'FAIL'", sql)
def test_ocr_permanent_counts_attempt(self):
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
self.assertIn("ocr_attempts + 1", sql)
self.assertIn("'FAIL'", sql)
class SubmitDedupeTests(unittest.TestCase):
def test_same_id_is_queued_once_until_done(self):
p = bare_pipeline()
pool = mock.Mock()
calls = []
p.ingest = lambda arg: calls.append(arg)
p.ingest.__name__ = "ingest"
p._submit(pool, p.ingest, "a")
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
p._submit(pool, p.ingest, "b")
self.assertEqual(pool.submit.call_count, 2)
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
p._guarded(token, fn, arg)
self.assertEqual(calls, ["a"])
p._submit(pool, p.ingest, "a")
self.assertEqual(pool.submit.call_count, 3)
def test_failing_worker_releases_the_id(self):
p = bare_pipeline()
pool = mock.Mock()
def boom(arg):
raise RuntimeError("x")
p._submit(pool, boom, "a")
token, fn, arg = pool.submit.call_args.args[1:4]
with mock.patch("builtins.print"):
p._guarded(token, fn, arg)
self.assertEqual(p._inflight, set())
class ReceiverStatusCodeTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
cls.port = cls.server.server_address[1]
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
cls.thread.start()
@classmethod
def tearDownClass(cls):
cls.server.shutdown()
cls.server.server_close()
def post(self, path, body, content_type):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
try:
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
resp = conn.getresponse()
return resp.status, resp.getheader("Retry-After"), resp.read()
finally:
conn.close()
def attachment(self):
boundary = "xx"
body = (
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
"Content-Type: image/jpeg\r\n\r\n"
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
path = f"/v2/events/{uuid.uuid4()}/attachments/"
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
status, retry_after, _ = self.attachment()
self.assertEqual(status, 503)
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
def test_unexpected_error_on_upload_stays_500(self):
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
status, retry_after, _ = self.attachment()
self.assertEqual(status, 500)
self.assertIsNone(retry_after)
def test_db_outage_on_event_gives_503(self):
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
mock.patch("builtins.print"):
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
self.assertEqual(status, 503)
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
def test_unknown_object_id_is_still_404(self):
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
status, _, _ = self.attachment()
self.assertEqual(status, 404)
if __name__ == "__main__":
unittest.main()
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet.
python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen
python -m tools.gundi_requeue --apply # setzt zurueck
python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778
Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue
- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3
(sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen.
- image ocr_state FAIL -> PENDING (ocr_attempts=0)
Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from gundi_pipeline import connect # noqa: E402
from main import load_config # noqa: E402
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)")
ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)")
ap.add_argument("--imei", help="nur diese IMEI (source)")
ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml"))
args = ap.parse_args()
cfg = load_config(args.config)
inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL"
image_where = "source = 'gundi' AND ocr_state = 'FAIL'"
params_inbox: list = []
params_image: list = []
if args.since:
inbox_where += " AND received_at >= %s"
image_where += " AND servertime >= %s"
params_inbox.append(args.since)
params_image.append(args.since)
if args.imei:
inbox_where += " AND source = %s"
image_where += " AND imei = %s"
params_inbox.append(args.imei)
params_image.append(args.imei)
with connect(cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox)
n_inbox = cur.fetchone()[0]
cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image)
n_image = cur.fetchone()[0]
print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}")
print(f"image OCR FAIL: {n_image}")
if not args.apply:
print("Trockenlauf - mit --apply zuruecksetzen.")
return 0
cur.execute(
f"""
UPDATE remote_cam.gundi_inbox
SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour'
WHERE {inbox_where}
""",
params_inbox,
)
print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege")
cur.execute(
f"""
UPDATE remote_cam.image
SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour'
WHERE {image_where}
""",
params_image,
)
print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege")
print("Der Sweep uebernimmt sie in der naechsten Runde.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env bash
# Live-Ueberwachung des Gundi-Push-Empfaengers: kommen ueberhaupt POSTs von ICU an?
#
# Verfolgt zwei Quellen parallel und faerbt die relevanten Zeilen ein:
# [app] docker logs des Receiver-Containers ([HTTP]-Requestzeilen, [GUNDI]-Events/Anhaenge, Fehler)
# [nginx] nginx-Access-Log (meles_dev3), gefiltert auf /v2/events; Fallback sudo, wenn nicht lesbar
# [traefik] Access-Log von Traefik (meles2), gefiltert auf /v2/events - zeigt auch Requests, die Traefik
# selbst beantwortet und die nie beim Container ankommen (404 = kein Router-Match,
# 502/504 = Container down, 308/301 durch http->https-Redirect ...)
#
# Voraussetzung fuer [traefik]: Access-Log in Traefik aktiviert (--accesslog=true). Liegt er
# als Datei im Container (--accesslog.filepath, hier /var/log/access.log), wird per docker exec
# mitgelesen; sonst per docker logs.
#
# Aufruf auf meles2:
# tools/watch_gundi.sh # live ab jetzt, inkl. der letzten 10 Minuten
# tools/watch_gundi.sh -s 2h # Rueckblick 2 Stunden, dann live
# tools/watch_gundi.sh -a # alle Zeilen statt nur POST/Fehler (inkl. /healthz)
# tools/watch_gundi.sh -t "" # ohne Traefik-Log
# tools/watch_gundi.sh -f /pfad/access.log # Traefik schreibt in eine Datei statt nach stdout
#
# Optionen:
# -n LOGFILE nginx-Access-Log (Default: /var/log/nginx/access.log wenn vorhanden, "" = aus)
# -c CONTAINER Receiver-Container (Default: melesicumover-gundi)
# -t CONTAINER Traefik-Container (Default: automatisch, erster Container mit "traefik" im Namen)
# -f LOGFILE Traefik-Access-Log als Datei statt docker logs
# -s SINCE docker-logs --since (Default: 10m)
# -a nichts filtern
set -u
CONTAINER="${GUNDI_CONTAINER:-melesicumover-gundi}"
NGINX_LOG="${GUNDI_NGINX_LOG-/var/log/nginx/access.log}"
TRAEFIK="${GUNDI_TRAEFIK:-auto}"
TRAEFIK_FILE=""
TRAEFIK_INNER_LOG="${GUNDI_TRAEFIK_LOG:-/var/log/access.log}"
SINCE="10m"
ALL=0
while getopts "c:n:t:f:s:ah" opt; do
case "$opt" in
c) CONTAINER="$OPTARG" ;;
n) NGINX_LOG="$OPTARG" ;;
t) TRAEFIK="$OPTARG" ;;
f) TRAEFIK_FILE="$OPTARG" ;;
s) SINCE="$OPTARG" ;;
a) ALL=1 ;;
*) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
esac
done
if ! docker inspect "$CONTAINER" >/dev/null 2>&1; then
echo "Container '$CONTAINER' nicht gefunden. Laufende Container:" >&2
docker ps --format ' {{.Names}}\t{{.Status}}' >&2
exit 1
fi
state=$(docker inspect -f '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}} started={{.State.StartedAt}}' "$CONTAINER")
echo "Container $CONTAINER: $state"
if curl -fsS -m 3 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
echo "Receiver lokal erreichbar (127.0.0.1:8090/healthz ok)"
else
echo "WARNUNG: 127.0.0.1:8090/healthz nicht erreichbar"
fi
echo "Strg+C beendet. Rueckblick: $SINCE"
echo "--------------------------------------------------------------------------------"
# Faerbt und filtert; gibt bei jedem Treffer den Zaehler mit aus.
colorize() {
awk -v all="$ALL" '
BEGIN { R="\033[31m"; G="\033[32m"; Y="\033[33m"; C="\033[36m"; B="\033[1m"; N="\033[0m"; posts=0 }
{
line = $0
is_post = (line ~ /"POST /)
is_gundi = (line ~ /\[GUNDI\]/)
is_err = (line ~ /(Traceback|Error|Exception|Fehler|WARN|ALARM|STOERUNG)/) || (line ~ /" (4[0-9][0-9]|5[0-9][0-9]) /)
if (!all && !is_post && !is_gundi && !is_err) next
if (is_post) posts++
col = ""
if (is_err) col = R
else if (is_gundi) col = G
else if (is_post) col = C
if (line ~ /" 30[18] /) col = Y # Redirect: Client folgt evtl. nicht mit POST
printf "%s%s%s", col, line, N
if (is_post) printf " %s(POST #%d)%s", B, posts, N
printf "\n"
fflush()
}'
}
pids=()
cleanup() { kill "${pids[@]}" 2>/dev/null; exit 0; }
trap cleanup INT TERM
docker logs -f -t --since "$SINCE" "$CONTAINER" 2>&1 \
| sed -u 's/^/[app] /' | colorize &
pids+=($!)
if [ -n "$NGINX_LOG" ] && [ -e "$NGINX_LOG" ]; then
# Auf meles_dev3 gehoert das Log www-data:adm (0640) - ohne adm-Gruppe per sudo lesen.
SUDO=""
if [ ! -r "$NGINX_LOG" ]; then
# sudo muss im Vordergrund nach dem Passwort fragen (in einer &-Pipeline haengt es);
# danach liest "sudo -n" aus dem Credential-Cache.
echo "nginx-Log nicht direkt lesbar - sudo-Passwort fuer das Mitlesen:"
if sudo -v; then
SUDO="sudo -n"
else
echo "Hinweis: kein sudo - nginx-Log wird nicht verfolgt" >&2
NGINX_LOG=""
fi
fi
if [ -n "$NGINX_LOG" ]; then
$SUDO tail -n 300 -F "$NGINX_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[nginx] /' | colorize &
pids+=($!)
fi
elif [ -n "$TRAEFIK_FILE" ]; then
if [ -r "$TRAEFIK_FILE" ]; then
tail -n 200 -F "$TRAEFIK_FILE" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[traefik] /' | colorize &
pids+=($!)
else
echo "Hinweis: $TRAEFIK_FILE nicht lesbar - Traefik-Log wird nicht verfolgt" >&2
fi
elif [ -n "$TRAEFIK" ]; then
if [ "$TRAEFIK" = "auto" ]; then
TRAEFIK=$(docker ps --format '{{.Names}}' | grep -i traefik | head -n1)
fi
if [ -n "$TRAEFIK" ] && docker inspect "$TRAEFIK" >/dev/null 2>&1; then
echo "Traefik-Container: $TRAEFIK (Access-Log: $TRAEFIK_INNER_LOG)"
# Traefik schreibt den Access-Log in eine Datei im Container (--accesslog.filepath),
# nicht nach stdout - daher docker exec tail. Fallback: docker logs.
if docker exec "$TRAEFIK" test -r "$TRAEFIK_INNER_LOG" 2>/dev/null; then
docker exec "$TRAEFIK" tail -n 300 -F "$TRAEFIK_INNER_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events|icu\.meles\.eu' | sed -u 's/^/[traefik] /' | colorize &
else
docker logs -f -t --since "$SINCE" "$TRAEFIK" 2>&1 | grep --line-buffered -E '/v2/events|icu\.meles\.eu|level=(error|fatal)' | sed -u 's/^/[traefik] /' | colorize &
fi
pids+=($!)
else
echo "Hinweis: kein Traefik-Container gefunden (-t NAME oder -f LOGFILE angeben)" >&2
fi
fi
wait