Make the Gundi receiver robust against S3/DB outages

- Upload request path uses its own S3 client with short timeouts (5s/30s,
  2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB
  failures answer 503 + Retry-After instead of 500 after minutes of waiting.
- Transient S3/DB errors in ingest/OCR no longer count against max_attempts
  (previously ~15 min of outage turned images into FAILED); the sweep retries.
- The sweep no longer queues the same id twice (no pile-up during long outages).
- Backlog check logs [GUNDI][ALARM] for stuck/failed entries.
- tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-05 16:40:48 +02:00
co-authored by Claude Opus 5.5
parent cb2566a537
commit 6d518b4542
7 changed files with 659 additions and 21 deletions
+252
View File
@@ -0,0 +1,252 @@
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
python -m unittest discover -s tests -v (oder: python -m pytest tests)
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
"""
import http.client
import json
import os
import sys
import threading
import unittest
import uuid
from types import SimpleNamespace
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import psycopg # noqa: E402
from botocore.exceptions import ClientError # noqa: E402
import gundi_pipeline as gp # noqa: E402
import gundi_receiver as gr # noqa: E402
def s3_unavailable() -> ClientError:
return ClientError(
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
"PutObject",
)
class FakeCursor:
def __init__(self, rows=None, rowcount=1):
self.executed: list[tuple[str, tuple]] = []
self._rows = list(rows or [])
self.rowcount = rowcount
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def execute(self, sql, params=()):
self.executed.append((" ".join(sql.split()), params))
def fetchone(self):
return self._rows.pop(0) if self._rows else None
class FakeConn:
def __init__(self, cursor: FakeCursor):
self._cursor = cursor
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def cursor(self):
return self._cursor
def fake_connect(cursor: FakeCursor):
return lambda dsn: FakeConn(cursor)
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
class AttachImageTests(unittest.TestCase):
def test_s3_down_raises_without_db_lock_and_without_update(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
s3 = mock.Mock()
s3.put_bytes.side_effect = s3_unavailable()
with mock.patch.object(gp, "connect", fake_connect(cur)):
with self.assertRaises(ClientError):
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
sqls = [sql for sql, _ in cur.executed]
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
def test_success_stores_then_updates(self):
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
def test_already_received_is_duplicate_without_s3(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
s3.put_bytes.assert_not_called()
def test_parallel_upload_lost_the_race_is_duplicate(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
def test_unknown_or_wrong_type_is_lookup_error(self):
for rows in ([], [("camera_status_report", "INGESTED")]):
cur = FakeCursor(rows=rows)
with mock.patch.object(gp, "connect", fake_connect(cur)):
with self.assertRaises(LookupError):
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
def bare_pipeline() -> gp.Pipeline:
p = gp.Pipeline.__new__(gp.Pipeline)
p.cfg = SimpleNamespace(pg_dsn="dsn")
p.gcfg = GCFG
p._inflight = set()
p._inflight_lock = threading.Lock()
return p
class FailureRecordingTests(unittest.TestCase):
def run_failure(self, method: str, err: Exception) -> str:
cur = FakeCursor(rows=[(1, "RECEIVED")])
p = bare_pipeline()
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
getattr(p, method)("id-1", err)
self.assertEqual(len(cur.executed), 1)
return cur.executed[0][0]
def test_ingest_transient_does_not_count_attempt(self):
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
self.assertNotIn("attempts + 1", sql)
self.assertNotIn("FAILED", sql)
def test_ingest_db_outage_does_not_count_attempt(self):
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
self.assertNotIn("attempts + 1", sql)
def test_ingest_permanent_counts_attempt(self):
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
self.assertIn("attempts + 1", sql)
self.assertIn("'FAILED'", sql)
def test_ocr_transient_does_not_count_attempt(self):
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
self.assertNotIn("ocr_attempts + 1", sql)
self.assertNotIn("'FAIL'", sql)
def test_ocr_permanent_counts_attempt(self):
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
self.assertIn("ocr_attempts + 1", sql)
self.assertIn("'FAIL'", sql)
class SubmitDedupeTests(unittest.TestCase):
def test_same_id_is_queued_once_until_done(self):
p = bare_pipeline()
pool = mock.Mock()
calls = []
p.ingest = lambda arg: calls.append(arg)
p.ingest.__name__ = "ingest"
p._submit(pool, p.ingest, "a")
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
p._submit(pool, p.ingest, "b")
self.assertEqual(pool.submit.call_count, 2)
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
p._guarded(token, fn, arg)
self.assertEqual(calls, ["a"])
p._submit(pool, p.ingest, "a")
self.assertEqual(pool.submit.call_count, 3)
def test_failing_worker_releases_the_id(self):
p = bare_pipeline()
pool = mock.Mock()
def boom(arg):
raise RuntimeError("x")
p._submit(pool, boom, "a")
token, fn, arg = pool.submit.call_args.args[1:4]
with mock.patch("builtins.print"):
p._guarded(token, fn, arg)
self.assertEqual(p._inflight, set())
class ReceiverStatusCodeTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
cls.port = cls.server.server_address[1]
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
cls.thread.start()
@classmethod
def tearDownClass(cls):
cls.server.shutdown()
cls.server.server_close()
def post(self, path, body, content_type):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
try:
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
resp = conn.getresponse()
return resp.status, resp.getheader("Retry-After"), resp.read()
finally:
conn.close()
def attachment(self):
boundary = "xx"
body = (
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
"Content-Type: image/jpeg\r\n\r\n"
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
path = f"/v2/events/{uuid.uuid4()}/attachments/"
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
status, retry_after, _ = self.attachment()
self.assertEqual(status, 503)
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
def test_unexpected_error_on_upload_stays_500(self):
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
status, retry_after, _ = self.attachment()
self.assertEqual(status, 500)
self.assertIsNone(retry_after)
def test_db_outage_on_event_gives_503(self):
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
mock.patch("builtins.print"):
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
self.assertEqual(status, 503)
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
def test_unknown_object_id_is_still_404(self):
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
status, _, _ = self.attachment()
self.assertEqual(status, 404)
if __name__ == "__main__":
unittest.main()