From 6d518b4542851b559286571c8f8e46f9ad5ad0d8 Mon Sep 17 00:00:00 2001 From: Dominik Dachs Date: Mon, 5 Oct 2026 16:40:48 +0200 Subject: [PATCH] Make the Gundi receiver robust against S3/DB outages - Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 --- README.md | 18 +++ config.yaml | 6 + gundi_pipeline.py | 158 +++++++++++++++++++-- gundi_receiver.py | 22 ++- tests/test_gundi_resilience.py | 252 +++++++++++++++++++++++++++++++++ tools/gundi_requeue.py | 82 +++++++++++ tools/watch_gundi.sh | 142 +++++++++++++++++++ 7 files changed, 659 insertions(+), 21 deletions(-) create mode 100644 tests/test_gundi_resilience.py create mode 100644 tools/gundi_requeue.py create mode 100644 tools/watch_gundi.sh diff --git a/README.md b/README.md index f4e9220..b0228be 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,24 @@ liest danach asynchron per OCR den Kamera-Zeitstempel, setzt `image.ocr_ts` und `DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert Events ohne Bild nach `abandon_after_hours` als `ABANDONED`. +### Verhalten bei S3-/DB-Ausfall + +| Teil | Verhalten | +|------|-----------| +| Status-Events, Event anlegen | brauchen kein S3; bei DB-Ausfall `503` + `Retry-After: 30` | +| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen | +| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` | +| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) | +| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren | + +Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`): + +```bash +docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue --apply +``` + +Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne Wiederholung geht das Bild verloren, wenn S3 laenger ausfaellt. + ### Datenmodell (`schema/001_domain_model.sql`) | Tabelle | Inhalt | diff --git a/config.yaml b/config.yaml index f93a923..b689b40 100644 --- a/config.yaml +++ b/config.yaml @@ -65,3 +65,9 @@ gundi: sweep_seconds: 60 stale_after_seconds: 300 max_attempts: 3 + # S3-Ausfall-Robustheit (siehe README): Upload-Request scheitert schnell mit 503 + Retry-After; + # S3-/DB-Stoerungen zaehlen nicht gegen max_attempts (nur Datenfehler). + upload_connect_timeout_seconds: 5 + upload_read_timeout_seconds: 30 + upload_max_attempts: 2 # inkl. erstem Versuch + alarm_after_seconds: 1800 # [GUNDI][ALARM], wenn Eintraege laenger haengen diff --git a/gundi_pipeline.py b/gundi_pipeline.py index 458b7b3..debff70 100644 --- a/gundi_pipeline.py +++ b/gundi_pipeline.py @@ -14,6 +14,7 @@ from typing import Optional import psycopg import yaml +from botocore.config import Config as BotoClientConfig from dateutil import tz from psycopg.types.json import Json @@ -21,6 +22,7 @@ from main import ( AppConfig, S3, exif_write_with_exiftool, + is_transient_error, make_thumbnail_bytes, ocr_extract_timestamp, to_exif_datetime, @@ -55,6 +57,12 @@ class GundiConfig: sweep_seconds: int = 60 stale_after_seconds: int = 300 max_attempts: int = 3 + # S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen + upload_connect_timeout_seconds: int = 5 + upload_read_timeout_seconds: int = 30 + upload_max_attempts: int = 2 # inkl. erstem Versuch + # Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen + alarm_after_seconds: int = 1800 def load_gundi_config(path: str) -> GundiConfig: @@ -82,6 +90,10 @@ def load_gundi_config(path: str) -> GundiConfig: sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)), stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)), max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))), + upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))), + upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))), + upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))), + alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))), ) if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN: raise SystemExit( @@ -329,28 +341,38 @@ def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]: def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str: - """Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.""" + """Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'. + + Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung + blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro + object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt. + """ with connect(dsn) as conn, conn.cursor() as cur: cur.execute( - "SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE", + "SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s", (object_id,), ) row = cur.fetchone() - if not row or row[0] not in EVENT_CAMERATRAP_TYPES: - raise LookupError(object_id) - if row[1] in ("RECEIVED", "INGESTED"): - return "duplicate" - key = f"{gcfg.inbox_prefix}{object_id}.jpg" - s3.put_bytes(key, jpg, "image/jpeg") + if not row or row[0] not in EVENT_CAMERATRAP_TYPES: + raise LookupError(object_id) + if row[1] in ("RECEIVED", "INGESTED"): + return "duplicate" + + key = f"{gcfg.inbox_prefix}{object_id}.jpg" + s3.put_bytes(key, jpg, "image/jpeg") + + with connect(dsn) as conn, conn.cursor() as cur: cur.execute( """ UPDATE remote_cam.gundi_inbox SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(), attempts = 0, last_error = NULL, updated_at = now() - WHERE object_id = %s + WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED') """, (key, object_id), ) + if cur.rowcount == 0: + return "duplicate" return "stored" @@ -384,10 +406,33 @@ class Pipeline: def __init__(self, cfg: AppConfig, gcfg: GundiConfig): self.cfg = cfg self.gcfg = gcfg - self.s3 = S3(cfg) + # Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell. + # Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt). + self.s3 = S3(cfg, self._s3_config( + cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds, + cfg.s3_max_attempts, cfg.s3_retry_mode, + )) + self.s3_request = S3(cfg, self._s3_config( + cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds, + gcfg.upload_max_attempts, "standard", + )) self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest") self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr") self._stop = threading.Event() + # Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf) + self._inflight: set[tuple[str, str]] = set() + self._inflight_lock = threading.Lock() + self._last_alarm = 0.0 + + @staticmethod + def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int, + max_attempts: int, retry_mode: str) -> BotoClientConfig: + return BotoClientConfig( + max_pool_connections=cfg.s3_max_pool_connections, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"mode": retry_mode, "total_max_attempts": max_attempts}, + ) def start(self) -> None: threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start() @@ -398,17 +443,32 @@ class Pipeline: self._ocr_pool.shutdown(wait=False, cancel_futures=True) def submit_ingest(self, object_id: str) -> None: - self._ingest_pool.submit(self._guarded, self.ingest, object_id) + self._submit(self._ingest_pool, self.ingest, object_id) def submit_ocr(self, image_uuid: str) -> None: - self._ocr_pool.submit(self._guarded, self.ocr, image_uuid) + self._submit(self._ocr_pool, self.ocr, image_uuid) - @staticmethod - def _guarded(fn, arg: str) -> None: + def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None: + token = (fn.__name__, arg) + with self._inflight_lock: + if token in self._inflight: + return + self._inflight.add(token) + try: + pool.submit(self._guarded, token, fn, arg) + except Exception: + with self._inflight_lock: + self._inflight.discard(token) + raise + + def _guarded(self, token: tuple[str, str], fn, arg: str) -> None: try: fn(arg) except Exception as e: print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True) + finally: + with self._inflight_lock: + self._inflight.discard(token) # --- Stufe 1 --------------------------------------------------------- def ingest(self, object_id: str) -> None: @@ -480,6 +540,20 @@ class Pipeline: def _record_ingest_failure(self, object_id: str, err: Exception) -> None: msg = f"{type(err).__name__}: {err}"[:1000] + if is_transient_error(err): + # S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min + # Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder. + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + UPDATE remote_cam.gundi_inbox + SET last_error = %s, updated_at = now() + WHERE object_id = %s AND state = 'RECEIVED' + """, + (msg, object_id), + ) + print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True) + return with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ @@ -578,6 +652,18 @@ class Pipeline: def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None: msg = f"{type(err).__name__}: {err}"[:1000] + if is_transient_error(err): + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + UPDATE remote_cam.image + SET ocr_error = %s, updated_at = now() + WHERE image_uuid = %s AND ocr_state = 'PENDING' + """, + (msg, image_uuid), + ) + print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True) + return with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( """ @@ -645,3 +731,47 @@ class Pipeline: f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}", flush=True, ) + self.check_backlog() + + def check_backlog(self) -> None: + """Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min).""" + limit = self.gcfg.alarm_after_seconds + with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT + count(*) FILTER (WHERE state = 'RECEIVED' + AND updated_at < now() - make_interval(secs => %s)), + count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT' + AND received_at < now() - make_interval(secs => %s)), + count(*) FILTER (WHERE state = 'FAILED') + FROM remote_cam.gundi_inbox + WHERE received_at > now() - interval '14 days' + """, + (limit, limit), + ) + received_stuck, awaiting_old, failed = cur.fetchone() + cur.execute( + """ + SELECT + count(*) FILTER (WHERE ocr_state = 'PENDING' + AND updated_at < now() - make_interval(secs => %s)), + count(*) FILTER (WHERE ocr_state = 'FAIL') + FROM remote_cam.image + WHERE source = 'gundi' AND servertime > now() - interval '14 days' + """, + (limit,), + ) + ocr_stuck, ocr_failed = cur.fetchone() + if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed): + return + now = time.monotonic() + if now - self._last_alarm < 300: + return + self._last_alarm = now + print( + f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} " + f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) " + "- bei failed/ocr_fail: tools/gundi_requeue.py", + flush=True, + ) diff --git a/gundi_receiver.py b/gundi_receiver.py index 999d49e..a5078fe 100644 --- a/gundi_receiver.py +++ b/gundi_receiver.py @@ -29,9 +29,10 @@ from gundi_pipeline import ( utc_now_str, validate_event, ) -from main import AppConfig, load_config +from main import AppConfig, is_transient_error, load_config EVENTS_PATH = "/v2/events/" +RETRY_AFTER_SECONDS = 30 ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$") JPEG_MAGIC = b"\xff\xd8\xff" @@ -73,18 +74,20 @@ class GundiHandler(BaseHTTPRequestHandler): print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True) # --- Antworten ------------------------------------------------------- - def _json(self, code: int, data) -> None: + def _json(self, code: int, data, *, retry_after: int | None = None) -> None: body = json.dumps(data, ensure_ascii=False).encode("utf-8") self.send_response(code) self.send_header("Content-Type", "application/json; charset=utf-8") + if retry_after is not None: + self.send_header("Retry-After", str(retry_after)) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) - def _error(self, code: int, message: str, *, close: bool = False) -> None: + def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None: if close: self.close_connection = True - self._json(code, {"error": message}) + self._json(code, {"error": message}, retry_after=retry_after) def _redirect(self, location: str) -> None: # 308 statt 301: der Client muss POST + Body beibehalten. @@ -155,8 +158,13 @@ class GundiHandler(BaseHTTPRequestHandler): else: self._handle_events(body) except Exception as e: - print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True) - self._error(500, "internal error", close=True) + if is_transient_error(e): + # S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt + print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True) + self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS) + else: + print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True) + self._error(500, "internal error", close=True) # --- Handler --------------------------------------------------------- def _handle_events(self, body: bytes) -> None: @@ -202,7 +210,7 @@ class GundiHandler(BaseHTTPRequestHandler): return try: outcome = attach_image( - self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg + self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg ) except LookupError: self._error(404, "Unknown object_id") diff --git a/tests/test_gundi_resilience.py b/tests/test_gundi_resilience.py new file mode 100644 index 0000000..a427118 --- /dev/null +++ b/tests/test_gundi_resilience.py @@ -0,0 +1,252 @@ +"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck. + + python -m unittest discover -s tests -v (oder: python -m pytest tests) + +Braucht die Pakete aus requirements.txt, aber weder Netz noch DB. +""" +import http.client +import json +import os +import sys +import threading +import unittest +import uuid +from types import SimpleNamespace +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import psycopg # noqa: E402 +from botocore.exceptions import ClientError # noqa: E402 + +import gundi_pipeline as gp # noqa: E402 +import gundi_receiver as gr # noqa: E402 + + +def s3_unavailable() -> ClientError: + return ClientError( + {"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, + "PutObject", + ) + + +class FakeCursor: + def __init__(self, rows=None, rowcount=1): + self.executed: list[tuple[str, tuple]] = [] + self._rows = list(rows or []) + self.rowcount = rowcount + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=()): + self.executed.append((" ".join(sql.split()), params)) + + def fetchone(self): + return self._rows.pop(0) if self._rows else None + + +class FakeConn: + def __init__(self, cursor: FakeCursor): + self._cursor = cursor + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def cursor(self): + return self._cursor + + +def fake_connect(cursor: FakeCursor): + return lambda dsn: FakeConn(cursor) + + +GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3) + + +class AttachImageTests(unittest.TestCase): + def test_s3_down_raises_without_db_lock_and_without_update(self): + cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")]) + s3 = mock.Mock() + s3.put_bytes.side_effect = s3_unavailable() + with mock.patch.object(gp, "connect", fake_connect(cur)): + with self.assertRaises(ClientError): + gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff") + sqls = [sql for sql, _ in cur.executed] + self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE + self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs + self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend + {"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject"))) + + def test_success_stores_then_updates(self): + cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")]) + s3 = mock.Mock() + with mock.patch.object(gp, "connect", fake_connect(cur)): + self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored") + s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg") + self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0]) + + def test_already_received_is_duplicate_without_s3(self): + cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")]) + s3 = mock.Mock() + with mock.patch.object(gp, "connect", fake_connect(cur)): + self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate") + s3.put_bytes.assert_not_called() + + def test_parallel_upload_lost_the_race_is_duplicate(self): + cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0) + with mock.patch.object(gp, "connect", fake_connect(cur)): + self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate") + + def test_unknown_or_wrong_type_is_lookup_error(self): + for rows in ([], [("camera_status_report", "INGESTED")]): + cur = FakeCursor(rows=rows) + with mock.patch.object(gp, "connect", fake_connect(cur)): + with self.assertRaises(LookupError): + gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg") + + +def bare_pipeline() -> gp.Pipeline: + p = gp.Pipeline.__new__(gp.Pipeline) + p.cfg = SimpleNamespace(pg_dsn="dsn") + p.gcfg = GCFG + p._inflight = set() + p._inflight_lock = threading.Lock() + return p + + +class FailureRecordingTests(unittest.TestCase): + def run_failure(self, method: str, err: Exception) -> str: + cur = FakeCursor(rows=[(1, "RECEIVED")]) + p = bare_pipeline() + with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"): + getattr(p, method)("id-1", err) + self.assertEqual(len(cur.executed), 1) + return cur.executed[0][0] + + def test_ingest_transient_does_not_count_attempt(self): + sql = self.run_failure("_record_ingest_failure", s3_unavailable()) + self.assertNotIn("attempts + 1", sql) + self.assertNotIn("FAILED", sql) + + def test_ingest_db_outage_does_not_count_attempt(self): + sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired")) + self.assertNotIn("attempts + 1", sql) + + def test_ingest_permanent_counts_attempt(self): + sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild")) + self.assertIn("attempts + 1", sql) + self.assertIn("'FAILED'", sql) + + def test_ocr_transient_does_not_count_attempt(self): + sql = self.run_failure("_record_ocr_failure", s3_unavailable()) + self.assertNotIn("ocr_attempts + 1", sql) + self.assertNotIn("'FAIL'", sql) + + def test_ocr_permanent_counts_attempt(self): + sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel")) + self.assertIn("ocr_attempts + 1", sql) + self.assertIn("'FAIL'", sql) + + +class SubmitDedupeTests(unittest.TestCase): + def test_same_id_is_queued_once_until_done(self): + p = bare_pipeline() + pool = mock.Mock() + calls = [] + p.ingest = lambda arg: calls.append(arg) + p.ingest.__name__ = "ingest" + + p._submit(pool, p.ingest, "a") + p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert + p._submit(pool, p.ingest, "b") + self.assertEqual(pool.submit.call_count, 2) + + # Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden + token, fn, arg = pool.submit.call_args_list[0].args[1:4] + p._guarded(token, fn, arg) + self.assertEqual(calls, ["a"]) + p._submit(pool, p.ingest, "a") + self.assertEqual(pool.submit.call_count, 3) + + def test_failing_worker_releases_the_id(self): + p = bare_pipeline() + pool = mock.Mock() + + def boom(arg): + raise RuntimeError("x") + + p._submit(pool, boom, "a") + token, fn, arg = pool.submit.call_args.args[1:4] + with mock.patch("builtins.print"): + p._guarded(token, fn, arg) + self.assertEqual(p._inflight, set()) + + +class ReceiverStatusCodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20) + cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock()) + cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline) + cls.port = cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + + def post(self, path, body, content_type): + conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) + try: + conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key}) + resp = conn.getresponse() + return resp.status, resp.getheader("Retry-After"), resp.read() + finally: + conn.close() + + def attachment(self): + boundary = "xx" + body = ( + f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n' + "Content-Type: image/jpeg\r\n\r\n" + ).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode() + path = f"/v2/events/{uuid.uuid4()}/attachments/" + return self.post(path, body, f"multipart/form-data; boundary={boundary}") + + def test_s3_outage_on_upload_gives_503_with_retry_after(self): + with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"): + status, retry_after, _ = self.attachment() + self.assertEqual(status, 503) + self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS)) + + def test_unexpected_error_on_upload_stays_500(self): + with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"): + status, retry_after, _ = self.attachment() + self.assertEqual(status, 500) + self.assertIsNone(retry_after) + + def test_db_outage_on_event_gives_503(self): + event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"} + with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \ + mock.patch("builtins.print"): + status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json") + self.assertEqual(status, 503) + self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS)) + + def test_unknown_object_id_is_still_404(self): + with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")): + status, _, _ = self.attachment() + self.assertEqual(status, 404) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/gundi_requeue.py b/tools/gundi_requeue.py new file mode 100644 index 0000000..31769aa --- /dev/null +++ b/tools/gundi_requeue.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet. + + python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen + python -m tools.gundi_requeue --apply # setzt zurueck + python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778 + +Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue + +- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3 + (sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen. +- image ocr_state FAIL -> PENDING (ocr_attempts=0) +Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds. +""" +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from gundi_pipeline import connect # noqa: E402 +from main import load_config # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)") + ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)") + ap.add_argument("--imei", help="nur diese IMEI (source)") + ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml")) + args = ap.parse_args() + + cfg = load_config(args.config) + inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL" + image_where = "source = 'gundi' AND ocr_state = 'FAIL'" + params_inbox: list = [] + params_image: list = [] + if args.since: + inbox_where += " AND received_at >= %s" + image_where += " AND servertime >= %s" + params_inbox.append(args.since) + params_image.append(args.since) + if args.imei: + inbox_where += " AND source = %s" + image_where += " AND imei = %s" + params_inbox.append(args.imei) + params_image.append(args.imei) + + with connect(cfg.pg_dsn) as conn, conn.cursor() as cur: + cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox) + n_inbox = cur.fetchone()[0] + cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image) + n_image = cur.fetchone()[0] + print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}") + print(f"image OCR FAIL: {n_image}") + if not args.apply: + print("Trockenlauf - mit --apply zuruecksetzen.") + return 0 + cur.execute( + f""" + UPDATE remote_cam.gundi_inbox + SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour' + WHERE {inbox_where} + """, + params_inbox, + ) + print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege") + cur.execute( + f""" + UPDATE remote_cam.image + SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour' + WHERE {image_where} + """, + params_image, + ) + print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege") + print("Der Sweep uebernimmt sie in der naechsten Runde.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/watch_gundi.sh b/tools/watch_gundi.sh new file mode 100644 index 0000000..741c60b --- /dev/null +++ b/tools/watch_gundi.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Live-Ueberwachung des Gundi-Push-Empfaengers: kommen ueberhaupt POSTs von ICU an? +# +# Verfolgt zwei Quellen parallel und faerbt die relevanten Zeilen ein: +# [app] docker logs des Receiver-Containers ([HTTP]-Requestzeilen, [GUNDI]-Events/Anhaenge, Fehler) +# [nginx] nginx-Access-Log (meles_dev3), gefiltert auf /v2/events; Fallback sudo, wenn nicht lesbar +# [traefik] Access-Log von Traefik (meles2), gefiltert auf /v2/events - zeigt auch Requests, die Traefik +# selbst beantwortet und die nie beim Container ankommen (404 = kein Router-Match, +# 502/504 = Container down, 308/301 durch http->https-Redirect ...) +# +# Voraussetzung fuer [traefik]: Access-Log in Traefik aktiviert (--accesslog=true). Liegt er +# als Datei im Container (--accesslog.filepath, hier /var/log/access.log), wird per docker exec +# mitgelesen; sonst per docker logs. +# +# Aufruf auf meles2: +# tools/watch_gundi.sh # live ab jetzt, inkl. der letzten 10 Minuten +# tools/watch_gundi.sh -s 2h # Rueckblick 2 Stunden, dann live +# tools/watch_gundi.sh -a # alle Zeilen statt nur POST/Fehler (inkl. /healthz) +# tools/watch_gundi.sh -t "" # ohne Traefik-Log +# tools/watch_gundi.sh -f /pfad/access.log # Traefik schreibt in eine Datei statt nach stdout +# +# Optionen: +# -n LOGFILE nginx-Access-Log (Default: /var/log/nginx/access.log wenn vorhanden, "" = aus) +# -c CONTAINER Receiver-Container (Default: melesicumover-gundi) +# -t CONTAINER Traefik-Container (Default: automatisch, erster Container mit "traefik" im Namen) +# -f LOGFILE Traefik-Access-Log als Datei statt docker logs +# -s SINCE docker-logs --since (Default: 10m) +# -a nichts filtern +set -u + +CONTAINER="${GUNDI_CONTAINER:-melesicumover-gundi}" +NGINX_LOG="${GUNDI_NGINX_LOG-/var/log/nginx/access.log}" +TRAEFIK="${GUNDI_TRAEFIK:-auto}" +TRAEFIK_FILE="" +TRAEFIK_INNER_LOG="${GUNDI_TRAEFIK_LOG:-/var/log/access.log}" +SINCE="10m" +ALL=0 + +while getopts "c:n:t:f:s:ah" opt; do + case "$opt" in + c) CONTAINER="$OPTARG" ;; + n) NGINX_LOG="$OPTARG" ;; + t) TRAEFIK="$OPTARG" ;; + f) TRAEFIK_FILE="$OPTARG" ;; + s) SINCE="$OPTARG" ;; + a) ALL=1 ;; + *) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + esac +done + +if ! docker inspect "$CONTAINER" >/dev/null 2>&1; then + echo "Container '$CONTAINER' nicht gefunden. Laufende Container:" >&2 + docker ps --format ' {{.Names}}\t{{.Status}}' >&2 + exit 1 +fi + +state=$(docker inspect -f '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}} started={{.State.StartedAt}}' "$CONTAINER") +echo "Container $CONTAINER: $state" +if curl -fsS -m 3 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then + echo "Receiver lokal erreichbar (127.0.0.1:8090/healthz ok)" +else + echo "WARNUNG: 127.0.0.1:8090/healthz nicht erreichbar" +fi +echo "Strg+C beendet. Rueckblick: $SINCE" +echo "--------------------------------------------------------------------------------" + +# Faerbt und filtert; gibt bei jedem Treffer den Zaehler mit aus. +colorize() { + awk -v all="$ALL" ' + BEGIN { R="\033[31m"; G="\033[32m"; Y="\033[33m"; C="\033[36m"; B="\033[1m"; N="\033[0m"; posts=0 } + { + line = $0 + is_post = (line ~ /"POST /) + is_gundi = (line ~ /\[GUNDI\]/) + is_err = (line ~ /(Traceback|Error|Exception|Fehler|WARN|ALARM|STOERUNG)/) || (line ~ /" (4[0-9][0-9]|5[0-9][0-9]) /) + if (!all && !is_post && !is_gundi && !is_err) next + if (is_post) posts++ + col = "" + if (is_err) col = R + else if (is_gundi) col = G + else if (is_post) col = C + if (line ~ /" 30[18] /) col = Y # Redirect: Client folgt evtl. nicht mit POST + printf "%s%s%s", col, line, N + if (is_post) printf " %s(POST #%d)%s", B, posts, N + printf "\n" + fflush() + }' +} + +pids=() +cleanup() { kill "${pids[@]}" 2>/dev/null; exit 0; } +trap cleanup INT TERM + +docker logs -f -t --since "$SINCE" "$CONTAINER" 2>&1 \ + | sed -u 's/^/[app] /' | colorize & +pids+=($!) + +if [ -n "$NGINX_LOG" ] && [ -e "$NGINX_LOG" ]; then + # Auf meles_dev3 gehoert das Log www-data:adm (0640) - ohne adm-Gruppe per sudo lesen. + SUDO="" + if [ ! -r "$NGINX_LOG" ]; then + # sudo muss im Vordergrund nach dem Passwort fragen (in einer &-Pipeline haengt es); + # danach liest "sudo -n" aus dem Credential-Cache. + echo "nginx-Log nicht direkt lesbar - sudo-Passwort fuer das Mitlesen:" + if sudo -v; then + SUDO="sudo -n" + else + echo "Hinweis: kein sudo - nginx-Log wird nicht verfolgt" >&2 + NGINX_LOG="" + fi + fi + if [ -n "$NGINX_LOG" ]; then + $SUDO tail -n 300 -F "$NGINX_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[nginx] /' | colorize & + pids+=($!) + fi +elif [ -n "$TRAEFIK_FILE" ]; then + if [ -r "$TRAEFIK_FILE" ]; then + tail -n 200 -F "$TRAEFIK_FILE" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[traefik] /' | colorize & + pids+=($!) + else + echo "Hinweis: $TRAEFIK_FILE nicht lesbar - Traefik-Log wird nicht verfolgt" >&2 + fi +elif [ -n "$TRAEFIK" ]; then + if [ "$TRAEFIK" = "auto" ]; then + TRAEFIK=$(docker ps --format '{{.Names}}' | grep -i traefik | head -n1) + fi + if [ -n "$TRAEFIK" ] && docker inspect "$TRAEFIK" >/dev/null 2>&1; then + echo "Traefik-Container: $TRAEFIK (Access-Log: $TRAEFIK_INNER_LOG)" + # Traefik schreibt den Access-Log in eine Datei im Container (--accesslog.filepath), + # nicht nach stdout - daher docker exec tail. Fallback: docker logs. + if docker exec "$TRAEFIK" test -r "$TRAEFIK_INNER_LOG" 2>/dev/null; then + docker exec "$TRAEFIK" tail -n 300 -F "$TRAEFIK_INNER_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events|icu\.meles\.eu' | sed -u 's/^/[traefik] /' | colorize & + else + docker logs -f -t --since "$SINCE" "$TRAEFIK" 2>&1 | grep --line-buffered -E '/v2/events|icu\.meles\.eu|level=(error|fatal)' | sed -u 's/^/[traefik] /' | colorize & + fi + pids+=($!) + else + echo "Hinweis: kein Traefik-Container gefunden (-t NAME oder -f LOGFILE angeben)" >&2 + fi +fi + +wait