Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,24 @@ liest danach asynchron per OCR den Kamera-Zeitstempel, setzt `image.ocr_ts` und
|
||||
`DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert
|
||||
Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
|
||||
|
||||
### Verhalten bei S3-/DB-Ausfall
|
||||
|
||||
| Teil | Verhalten |
|
||||
|------|-----------|
|
||||
| Status-Events, Event anlegen | brauchen kein S3; bei DB-Ausfall `503` + `Retry-After: 30` |
|
||||
| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen |
|
||||
| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` |
|
||||
| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) |
|
||||
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren |
|
||||
|
||||
Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`):
|
||||
|
||||
```bash
|
||||
docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue --apply
|
||||
```
|
||||
|
||||
Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne Wiederholung geht das Bild verloren, wenn S3 laenger ausfaellt.
|
||||
|
||||
### Datenmodell (`schema/001_domain_model.sql`)
|
||||
|
||||
| Tabelle | Inhalt |
|
||||
|
||||
@@ -65,3 +65,9 @@ gundi:
|
||||
sweep_seconds: 60
|
||||
stale_after_seconds: 300
|
||||
max_attempts: 3
|
||||
# S3-Ausfall-Robustheit (siehe README): Upload-Request scheitert schnell mit 503 + Retry-After;
|
||||
# S3-/DB-Stoerungen zaehlen nicht gegen max_attempts (nur Datenfehler).
|
||||
upload_connect_timeout_seconds: 5
|
||||
upload_read_timeout_seconds: 30
|
||||
upload_max_attempts: 2 # inkl. erstem Versuch
|
||||
alarm_after_seconds: 1800 # [GUNDI][ALARM], wenn Eintraege laenger haengen
|
||||
|
||||
+138
-8
@@ -14,6 +14,7 @@ from typing import Optional
|
||||
|
||||
import psycopg
|
||||
import yaml
|
||||
from botocore.config import Config as BotoClientConfig
|
||||
from dateutil import tz
|
||||
from psycopg.types.json import Json
|
||||
|
||||
@@ -21,6 +22,7 @@ from main import (
|
||||
AppConfig,
|
||||
S3,
|
||||
exif_write_with_exiftool,
|
||||
is_transient_error,
|
||||
make_thumbnail_bytes,
|
||||
ocr_extract_timestamp,
|
||||
to_exif_datetime,
|
||||
@@ -55,6 +57,12 @@ class GundiConfig:
|
||||
sweep_seconds: int = 60
|
||||
stale_after_seconds: int = 300
|
||||
max_attempts: int = 3
|
||||
# S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen
|
||||
upload_connect_timeout_seconds: int = 5
|
||||
upload_read_timeout_seconds: int = 30
|
||||
upload_max_attempts: int = 2 # inkl. erstem Versuch
|
||||
# Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen
|
||||
alarm_after_seconds: int = 1800
|
||||
|
||||
|
||||
def load_gundi_config(path: str) -> GundiConfig:
|
||||
@@ -82,6 +90,10 @@ def load_gundi_config(path: str) -> GundiConfig:
|
||||
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
|
||||
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
|
||||
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
|
||||
upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))),
|
||||
upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))),
|
||||
upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))),
|
||||
alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))),
|
||||
)
|
||||
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
|
||||
raise SystemExit(
|
||||
@@ -329,10 +341,15 @@ def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
|
||||
|
||||
|
||||
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
|
||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'."""
|
||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.
|
||||
|
||||
Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung
|
||||
blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro
|
||||
object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt.
|
||||
"""
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE",
|
||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s",
|
||||
(object_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
@@ -340,17 +357,22 @@ def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes
|
||||
raise LookupError(object_id)
|
||||
if row[1] in ("RECEIVED", "INGESTED"):
|
||||
return "duplicate"
|
||||
|
||||
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
||||
s3.put_bytes(key, jpg, "image/jpeg")
|
||||
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
|
||||
attempts = 0, last_error = NULL, updated_at = now()
|
||||
WHERE object_id = %s
|
||||
WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED')
|
||||
""",
|
||||
(key, object_id),
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
return "duplicate"
|
||||
return "stored"
|
||||
|
||||
|
||||
@@ -384,10 +406,33 @@ class Pipeline:
|
||||
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
|
||||
self.cfg = cfg
|
||||
self.gcfg = gcfg
|
||||
self.s3 = S3(cfg)
|
||||
# Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell.
|
||||
# Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt).
|
||||
self.s3 = S3(cfg, self._s3_config(
|
||||
cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds,
|
||||
cfg.s3_max_attempts, cfg.s3_retry_mode,
|
||||
))
|
||||
self.s3_request = S3(cfg, self._s3_config(
|
||||
cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds,
|
||||
gcfg.upload_max_attempts, "standard",
|
||||
))
|
||||
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
|
||||
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
|
||||
self._stop = threading.Event()
|
||||
# Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf)
|
||||
self._inflight: set[tuple[str, str]] = set()
|
||||
self._inflight_lock = threading.Lock()
|
||||
self._last_alarm = 0.0
|
||||
|
||||
@staticmethod
|
||||
def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int,
|
||||
max_attempts: int, retry_mode: str) -> BotoClientConfig:
|
||||
return BotoClientConfig(
|
||||
max_pool_connections=cfg.s3_max_pool_connections,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"mode": retry_mode, "total_max_attempts": max_attempts},
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
|
||||
@@ -398,17 +443,32 @@ class Pipeline:
|
||||
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_ingest(self, object_id: str) -> None:
|
||||
self._ingest_pool.submit(self._guarded, self.ingest, object_id)
|
||||
self._submit(self._ingest_pool, self.ingest, object_id)
|
||||
|
||||
def submit_ocr(self, image_uuid: str) -> None:
|
||||
self._ocr_pool.submit(self._guarded, self.ocr, image_uuid)
|
||||
self._submit(self._ocr_pool, self.ocr, image_uuid)
|
||||
|
||||
@staticmethod
|
||||
def _guarded(fn, arg: str) -> None:
|
||||
def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None:
|
||||
token = (fn.__name__, arg)
|
||||
with self._inflight_lock:
|
||||
if token in self._inflight:
|
||||
return
|
||||
self._inflight.add(token)
|
||||
try:
|
||||
pool.submit(self._guarded, token, fn, arg)
|
||||
except Exception:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(token)
|
||||
raise
|
||||
|
||||
def _guarded(self, token: tuple[str, str], fn, arg: str) -> None:
|
||||
try:
|
||||
fn(arg)
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(token)
|
||||
|
||||
# --- Stufe 1 ---------------------------------------------------------
|
||||
def ingest(self, object_id: str) -> None:
|
||||
@@ -480,6 +540,20 @@ class Pipeline:
|
||||
|
||||
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
if is_transient_error(err):
|
||||
# S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min
|
||||
# Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder.
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET last_error = %s, updated_at = now()
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
""",
|
||||
(msg, object_id),
|
||||
)
|
||||
print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||
return
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
@@ -578,6 +652,18 @@ class Pipeline:
|
||||
|
||||
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
if is_transient_error(err):
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
""",
|
||||
(msg, image_uuid),
|
||||
)
|
||||
print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||
return
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
@@ -645,3 +731,47 @@ class Pipeline:
|
||||
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
|
||||
flush=True,
|
||||
)
|
||||
self.check_backlog()
|
||||
|
||||
def check_backlog(self) -> None:
|
||||
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
|
||||
limit = self.gcfg.alarm_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE state = 'RECEIVED'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'FAILED')
|
||||
FROM remote_cam.gundi_inbox
|
||||
WHERE received_at > now() - interval '14 days'
|
||||
""",
|
||||
(limit, limit),
|
||||
)
|
||||
received_stuck, awaiting_old, failed = cur.fetchone()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE ocr_state = 'PENDING'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL')
|
||||
FROM remote_cam.image
|
||||
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
ocr_stuck, ocr_failed = cur.fetchone()
|
||||
if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed):
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now - self._last_alarm < 300:
|
||||
return
|
||||
self._last_alarm = now
|
||||
print(
|
||||
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
|
||||
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
|
||||
"- bei failed/ocr_fail: tools/gundi_requeue.py",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
+13
-5
@@ -29,9 +29,10 @@ from gundi_pipeline import (
|
||||
utc_now_str,
|
||||
validate_event,
|
||||
)
|
||||
from main import AppConfig, load_config
|
||||
from main import AppConfig, is_transient_error, load_config
|
||||
|
||||
EVENTS_PATH = "/v2/events/"
|
||||
RETRY_AFTER_SECONDS = 30
|
||||
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
@@ -73,18 +74,20 @@ class GundiHandler(BaseHTTPRequestHandler):
|
||||
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
|
||||
|
||||
# --- Antworten -------------------------------------------------------
|
||||
def _json(self, code: int, data) -> None:
|
||||
def _json(self, code: int, data, *, retry_after: int | None = None) -> None:
|
||||
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
if retry_after is not None:
|
||||
self.send_header("Retry-After", str(retry_after))
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _error(self, code: int, message: str, *, close: bool = False) -> None:
|
||||
def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None:
|
||||
if close:
|
||||
self.close_connection = True
|
||||
self._json(code, {"error": message})
|
||||
self._json(code, {"error": message}, retry_after=retry_after)
|
||||
|
||||
def _redirect(self, location: str) -> None:
|
||||
# 308 statt 301: der Client muss POST + Body beibehalten.
|
||||
@@ -155,6 +158,11 @@ class GundiHandler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
self._handle_events(body)
|
||||
except Exception as e:
|
||||
if is_transient_error(e):
|
||||
# S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt
|
||||
print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS)
|
||||
else:
|
||||
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(500, "internal error", close=True)
|
||||
|
||||
@@ -202,7 +210,7 @@ class GundiHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
try:
|
||||
outcome = attach_image(
|
||||
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg
|
||||
self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg
|
||||
)
|
||||
except LookupError:
|
||||
self._error(404, "Unknown object_id")
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
|
||||
|
||||
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
||||
|
||||
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
||||
"""
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import psycopg # noqa: E402
|
||||
from botocore.exceptions import ClientError # noqa: E402
|
||||
|
||||
import gundi_pipeline as gp # noqa: E402
|
||||
import gundi_receiver as gr # noqa: E402
|
||||
|
||||
|
||||
def s3_unavailable() -> ClientError:
|
||||
return ClientError(
|
||||
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
|
||||
"PutObject",
|
||||
)
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, rows=None, rowcount=1):
|
||||
self.executed: list[tuple[str, tuple]] = []
|
||||
self._rows = list(rows or [])
|
||||
self.rowcount = rowcount
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
self.executed.append((" ".join(sql.split()), params))
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows.pop(0) if self._rows else None
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, cursor: FakeCursor):
|
||||
self._cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self._cursor
|
||||
|
||||
|
||||
def fake_connect(cursor: FakeCursor):
|
||||
return lambda dsn: FakeConn(cursor)
|
||||
|
||||
|
||||
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
|
||||
|
||||
|
||||
class AttachImageTests(unittest.TestCase):
|
||||
def test_s3_down_raises_without_db_lock_and_without_update(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||
s3 = mock.Mock()
|
||||
s3.put_bytes.side_effect = s3_unavailable()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
with self.assertRaises(ClientError):
|
||||
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
|
||||
sqls = [sql for sql, _ in cur.executed]
|
||||
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
|
||||
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
|
||||
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
|
||||
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
|
||||
|
||||
def test_success_stores_then_updates(self):
|
||||
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
||||
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
||||
|
||||
def test_already_received_is_duplicate_without_s3(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
|
||||
s3.put_bytes.assert_not_called()
|
||||
|
||||
def test_parallel_upload_lost_the_race_is_duplicate(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
|
||||
|
||||
def test_unknown_or_wrong_type_is_lookup_error(self):
|
||||
for rows in ([], [("camera_status_report", "INGESTED")]):
|
||||
cur = FakeCursor(rows=rows)
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
with self.assertRaises(LookupError):
|
||||
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
|
||||
|
||||
|
||||
def bare_pipeline() -> gp.Pipeline:
|
||||
p = gp.Pipeline.__new__(gp.Pipeline)
|
||||
p.cfg = SimpleNamespace(pg_dsn="dsn")
|
||||
p.gcfg = GCFG
|
||||
p._inflight = set()
|
||||
p._inflight_lock = threading.Lock()
|
||||
return p
|
||||
|
||||
|
||||
class FailureRecordingTests(unittest.TestCase):
|
||||
def run_failure(self, method: str, err: Exception) -> str:
|
||||
cur = FakeCursor(rows=[(1, "RECEIVED")])
|
||||
p = bare_pipeline()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
|
||||
getattr(p, method)("id-1", err)
|
||||
self.assertEqual(len(cur.executed), 1)
|
||||
return cur.executed[0][0]
|
||||
|
||||
def test_ingest_transient_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
|
||||
self.assertNotIn("attempts + 1", sql)
|
||||
self.assertNotIn("FAILED", sql)
|
||||
|
||||
def test_ingest_db_outage_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
|
||||
self.assertNotIn("attempts + 1", sql)
|
||||
|
||||
def test_ingest_permanent_counts_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
|
||||
self.assertIn("attempts + 1", sql)
|
||||
self.assertIn("'FAILED'", sql)
|
||||
|
||||
def test_ocr_transient_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
|
||||
self.assertNotIn("ocr_attempts + 1", sql)
|
||||
self.assertNotIn("'FAIL'", sql)
|
||||
|
||||
def test_ocr_permanent_counts_attempt(self):
|
||||
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
|
||||
self.assertIn("ocr_attempts + 1", sql)
|
||||
self.assertIn("'FAIL'", sql)
|
||||
|
||||
|
||||
class SubmitDedupeTests(unittest.TestCase):
|
||||
def test_same_id_is_queued_once_until_done(self):
|
||||
p = bare_pipeline()
|
||||
pool = mock.Mock()
|
||||
calls = []
|
||||
p.ingest = lambda arg: calls.append(arg)
|
||||
p.ingest.__name__ = "ingest"
|
||||
|
||||
p._submit(pool, p.ingest, "a")
|
||||
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
|
||||
p._submit(pool, p.ingest, "b")
|
||||
self.assertEqual(pool.submit.call_count, 2)
|
||||
|
||||
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
|
||||
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
|
||||
p._guarded(token, fn, arg)
|
||||
self.assertEqual(calls, ["a"])
|
||||
p._submit(pool, p.ingest, "a")
|
||||
self.assertEqual(pool.submit.call_count, 3)
|
||||
|
||||
def test_failing_worker_releases_the_id(self):
|
||||
p = bare_pipeline()
|
||||
pool = mock.Mock()
|
||||
|
||||
def boom(arg):
|
||||
raise RuntimeError("x")
|
||||
|
||||
p._submit(pool, boom, "a")
|
||||
token, fn, arg = pool.submit.call_args.args[1:4]
|
||||
with mock.patch("builtins.print"):
|
||||
p._guarded(token, fn, arg)
|
||||
self.assertEqual(p._inflight, set())
|
||||
|
||||
|
||||
class ReceiverStatusCodeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
|
||||
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
|
||||
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
|
||||
cls.port = cls.server.server_address[1]
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
||||
cls.thread.start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.shutdown()
|
||||
cls.server.server_close()
|
||||
|
||||
def post(self, path, body, content_type):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
try:
|
||||
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
|
||||
resp = conn.getresponse()
|
||||
return resp.status, resp.getheader("Retry-After"), resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def attachment(self):
|
||||
boundary = "xx"
|
||||
body = (
|
||||
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
|
||||
"Content-Type: image/jpeg\r\n\r\n"
|
||||
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
|
||||
path = f"/v2/events/{uuid.uuid4()}/attachments/"
|
||||
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
|
||||
|
||||
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
|
||||
status, retry_after, _ = self.attachment()
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||
|
||||
def test_unexpected_error_on_upload_stays_500(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
|
||||
status, retry_after, _ = self.attachment()
|
||||
self.assertEqual(status, 500)
|
||||
self.assertIsNone(retry_after)
|
||||
|
||||
def test_db_outage_on_event_gives_503(self):
|
||||
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
|
||||
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
|
||||
mock.patch("builtins.print"):
|
||||
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||
|
||||
def test_unknown_object_id_is_still_404(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
|
||||
status, _, _ = self.attachment()
|
||||
self.assertEqual(status, 404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet.
|
||||
|
||||
python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen
|
||||
python -m tools.gundi_requeue --apply # setzt zurueck
|
||||
python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778
|
||||
|
||||
Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue
|
||||
|
||||
- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3
|
||||
(sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen.
|
||||
- image ocr_state FAIL -> PENDING (ocr_attempts=0)
|
||||
Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from gundi_pipeline import connect # noqa: E402
|
||||
from main import load_config # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)")
|
||||
ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)")
|
||||
ap.add_argument("--imei", help="nur diese IMEI (source)")
|
||||
ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL"
|
||||
image_where = "source = 'gundi' AND ocr_state = 'FAIL'"
|
||||
params_inbox: list = []
|
||||
params_image: list = []
|
||||
if args.since:
|
||||
inbox_where += " AND received_at >= %s"
|
||||
image_where += " AND servertime >= %s"
|
||||
params_inbox.append(args.since)
|
||||
params_image.append(args.since)
|
||||
if args.imei:
|
||||
inbox_where += " AND source = %s"
|
||||
image_where += " AND imei = %s"
|
||||
params_inbox.append(args.imei)
|
||||
params_image.append(args.imei)
|
||||
|
||||
with connect(cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox)
|
||||
n_inbox = cur.fetchone()[0]
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image)
|
||||
n_image = cur.fetchone()[0]
|
||||
print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}")
|
||||
print(f"image OCR FAIL: {n_image}")
|
||||
if not args.apply:
|
||||
print("Trockenlauf - mit --apply zuruecksetzen.")
|
||||
return 0
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {inbox_where}
|
||||
""",
|
||||
params_inbox,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege")
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {image_where}
|
||||
""",
|
||||
params_image,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege")
|
||||
print("Der Sweep uebernimmt sie in der naechsten Runde.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
# Live-Ueberwachung des Gundi-Push-Empfaengers: kommen ueberhaupt POSTs von ICU an?
|
||||
#
|
||||
# Verfolgt zwei Quellen parallel und faerbt die relevanten Zeilen ein:
|
||||
# [app] docker logs des Receiver-Containers ([HTTP]-Requestzeilen, [GUNDI]-Events/Anhaenge, Fehler)
|
||||
# [nginx] nginx-Access-Log (meles_dev3), gefiltert auf /v2/events; Fallback sudo, wenn nicht lesbar
|
||||
# [traefik] Access-Log von Traefik (meles2), gefiltert auf /v2/events - zeigt auch Requests, die Traefik
|
||||
# selbst beantwortet und die nie beim Container ankommen (404 = kein Router-Match,
|
||||
# 502/504 = Container down, 308/301 durch http->https-Redirect ...)
|
||||
#
|
||||
# Voraussetzung fuer [traefik]: Access-Log in Traefik aktiviert (--accesslog=true). Liegt er
|
||||
# als Datei im Container (--accesslog.filepath, hier /var/log/access.log), wird per docker exec
|
||||
# mitgelesen; sonst per docker logs.
|
||||
#
|
||||
# Aufruf auf meles2:
|
||||
# tools/watch_gundi.sh # live ab jetzt, inkl. der letzten 10 Minuten
|
||||
# tools/watch_gundi.sh -s 2h # Rueckblick 2 Stunden, dann live
|
||||
# tools/watch_gundi.sh -a # alle Zeilen statt nur POST/Fehler (inkl. /healthz)
|
||||
# tools/watch_gundi.sh -t "" # ohne Traefik-Log
|
||||
# tools/watch_gundi.sh -f /pfad/access.log # Traefik schreibt in eine Datei statt nach stdout
|
||||
#
|
||||
# Optionen:
|
||||
# -n LOGFILE nginx-Access-Log (Default: /var/log/nginx/access.log wenn vorhanden, "" = aus)
|
||||
# -c CONTAINER Receiver-Container (Default: melesicumover-gundi)
|
||||
# -t CONTAINER Traefik-Container (Default: automatisch, erster Container mit "traefik" im Namen)
|
||||
# -f LOGFILE Traefik-Access-Log als Datei statt docker logs
|
||||
# -s SINCE docker-logs --since (Default: 10m)
|
||||
# -a nichts filtern
|
||||
set -u
|
||||
|
||||
CONTAINER="${GUNDI_CONTAINER:-melesicumover-gundi}"
|
||||
NGINX_LOG="${GUNDI_NGINX_LOG-/var/log/nginx/access.log}"
|
||||
TRAEFIK="${GUNDI_TRAEFIK:-auto}"
|
||||
TRAEFIK_FILE=""
|
||||
TRAEFIK_INNER_LOG="${GUNDI_TRAEFIK_LOG:-/var/log/access.log}"
|
||||
SINCE="10m"
|
||||
ALL=0
|
||||
|
||||
while getopts "c:n:t:f:s:ah" opt; do
|
||||
case "$opt" in
|
||||
c) CONTAINER="$OPTARG" ;;
|
||||
n) NGINX_LOG="$OPTARG" ;;
|
||||
t) TRAEFIK="$OPTARG" ;;
|
||||
f) TRAEFIK_FILE="$OPTARG" ;;
|
||||
s) SINCE="$OPTARG" ;;
|
||||
a) ALL=1 ;;
|
||||
*) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! docker inspect "$CONTAINER" >/dev/null 2>&1; then
|
||||
echo "Container '$CONTAINER' nicht gefunden. Laufende Container:" >&2
|
||||
docker ps --format ' {{.Names}}\t{{.Status}}' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
state=$(docker inspect -f '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}} started={{.State.StartedAt}}' "$CONTAINER")
|
||||
echo "Container $CONTAINER: $state"
|
||||
if curl -fsS -m 3 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
|
||||
echo "Receiver lokal erreichbar (127.0.0.1:8090/healthz ok)"
|
||||
else
|
||||
echo "WARNUNG: 127.0.0.1:8090/healthz nicht erreichbar"
|
||||
fi
|
||||
echo "Strg+C beendet. Rueckblick: $SINCE"
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
|
||||
# Faerbt und filtert; gibt bei jedem Treffer den Zaehler mit aus.
|
||||
colorize() {
|
||||
awk -v all="$ALL" '
|
||||
BEGIN { R="\033[31m"; G="\033[32m"; Y="\033[33m"; C="\033[36m"; B="\033[1m"; N="\033[0m"; posts=0 }
|
||||
{
|
||||
line = $0
|
||||
is_post = (line ~ /"POST /)
|
||||
is_gundi = (line ~ /\[GUNDI\]/)
|
||||
is_err = (line ~ /(Traceback|Error|Exception|Fehler|WARN|ALARM|STOERUNG)/) || (line ~ /" (4[0-9][0-9]|5[0-9][0-9]) /)
|
||||
if (!all && !is_post && !is_gundi && !is_err) next
|
||||
if (is_post) posts++
|
||||
col = ""
|
||||
if (is_err) col = R
|
||||
else if (is_gundi) col = G
|
||||
else if (is_post) col = C
|
||||
if (line ~ /" 30[18] /) col = Y # Redirect: Client folgt evtl. nicht mit POST
|
||||
printf "%s%s%s", col, line, N
|
||||
if (is_post) printf " %s(POST #%d)%s", B, posts, N
|
||||
printf "\n"
|
||||
fflush()
|
||||
}'
|
||||
}
|
||||
|
||||
pids=()
|
||||
cleanup() { kill "${pids[@]}" 2>/dev/null; exit 0; }
|
||||
trap cleanup INT TERM
|
||||
|
||||
docker logs -f -t --since "$SINCE" "$CONTAINER" 2>&1 \
|
||||
| sed -u 's/^/[app] /' | colorize &
|
||||
pids+=($!)
|
||||
|
||||
if [ -n "$NGINX_LOG" ] && [ -e "$NGINX_LOG" ]; then
|
||||
# Auf meles_dev3 gehoert das Log www-data:adm (0640) - ohne adm-Gruppe per sudo lesen.
|
||||
SUDO=""
|
||||
if [ ! -r "$NGINX_LOG" ]; then
|
||||
# sudo muss im Vordergrund nach dem Passwort fragen (in einer &-Pipeline haengt es);
|
||||
# danach liest "sudo -n" aus dem Credential-Cache.
|
||||
echo "nginx-Log nicht direkt lesbar - sudo-Passwort fuer das Mitlesen:"
|
||||
if sudo -v; then
|
||||
SUDO="sudo -n"
|
||||
else
|
||||
echo "Hinweis: kein sudo - nginx-Log wird nicht verfolgt" >&2
|
||||
NGINX_LOG=""
|
||||
fi
|
||||
fi
|
||||
if [ -n "$NGINX_LOG" ]; then
|
||||
$SUDO tail -n 300 -F "$NGINX_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[nginx] /' | colorize &
|
||||
pids+=($!)
|
||||
fi
|
||||
elif [ -n "$TRAEFIK_FILE" ]; then
|
||||
if [ -r "$TRAEFIK_FILE" ]; then
|
||||
tail -n 200 -F "$TRAEFIK_FILE" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[traefik] /' | colorize &
|
||||
pids+=($!)
|
||||
else
|
||||
echo "Hinweis: $TRAEFIK_FILE nicht lesbar - Traefik-Log wird nicht verfolgt" >&2
|
||||
fi
|
||||
elif [ -n "$TRAEFIK" ]; then
|
||||
if [ "$TRAEFIK" = "auto" ]; then
|
||||
TRAEFIK=$(docker ps --format '{{.Names}}' | grep -i traefik | head -n1)
|
||||
fi
|
||||
if [ -n "$TRAEFIK" ] && docker inspect "$TRAEFIK" >/dev/null 2>&1; then
|
||||
echo "Traefik-Container: $TRAEFIK (Access-Log: $TRAEFIK_INNER_LOG)"
|
||||
# Traefik schreibt den Access-Log in eine Datei im Container (--accesslog.filepath),
|
||||
# nicht nach stdout - daher docker exec tail. Fallback: docker logs.
|
||||
if docker exec "$TRAEFIK" test -r "$TRAEFIK_INNER_LOG" 2>/dev/null; then
|
||||
docker exec "$TRAEFIK" tail -n 300 -F "$TRAEFIK_INNER_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events|icu\.meles\.eu' | sed -u 's/^/[traefik] /' | colorize &
|
||||
else
|
||||
docker logs -f -t --since "$SINCE" "$TRAEFIK" 2>&1 | grep --line-buffered -E '/v2/events|icu\.meles\.eu|level=(error|fatal)' | sed -u 's/^/[traefik] /' | colorize &
|
||||
fi
|
||||
pids+=($!)
|
||||
else
|
||||
echo "Hinweis: kein Traefik-Container gefunden (-t NAME oder -f LOGFILE angeben)" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
wait
|
||||
Reference in New Issue
Block a user