Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,252 @@
|
||||
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
|
||||
|
||||
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
||||
|
||||
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
||||
"""
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import psycopg # noqa: E402
|
||||
from botocore.exceptions import ClientError # noqa: E402
|
||||
|
||||
import gundi_pipeline as gp # noqa: E402
|
||||
import gundi_receiver as gr # noqa: E402
|
||||
|
||||
|
||||
def s3_unavailable() -> ClientError:
|
||||
return ClientError(
|
||||
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
|
||||
"PutObject",
|
||||
)
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, rows=None, rowcount=1):
|
||||
self.executed: list[tuple[str, tuple]] = []
|
||||
self._rows = list(rows or [])
|
||||
self.rowcount = rowcount
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
self.executed.append((" ".join(sql.split()), params))
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows.pop(0) if self._rows else None
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, cursor: FakeCursor):
|
||||
self._cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self._cursor
|
||||
|
||||
|
||||
def fake_connect(cursor: FakeCursor):
|
||||
return lambda dsn: FakeConn(cursor)
|
||||
|
||||
|
||||
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
|
||||
|
||||
|
||||
class AttachImageTests(unittest.TestCase):
|
||||
def test_s3_down_raises_without_db_lock_and_without_update(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||
s3 = mock.Mock()
|
||||
s3.put_bytes.side_effect = s3_unavailable()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
with self.assertRaises(ClientError):
|
||||
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
|
||||
sqls = [sql for sql, _ in cur.executed]
|
||||
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
|
||||
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
|
||||
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
|
||||
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
|
||||
|
||||
def test_success_stores_then_updates(self):
|
||||
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
||||
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
||||
|
||||
def test_already_received_is_duplicate_without_s3(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
|
||||
s3.put_bytes.assert_not_called()
|
||||
|
||||
def test_parallel_upload_lost_the_race_is_duplicate(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
|
||||
|
||||
def test_unknown_or_wrong_type_is_lookup_error(self):
|
||||
for rows in ([], [("camera_status_report", "INGESTED")]):
|
||||
cur = FakeCursor(rows=rows)
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
with self.assertRaises(LookupError):
|
||||
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
|
||||
|
||||
|
||||
def bare_pipeline() -> gp.Pipeline:
|
||||
p = gp.Pipeline.__new__(gp.Pipeline)
|
||||
p.cfg = SimpleNamespace(pg_dsn="dsn")
|
||||
p.gcfg = GCFG
|
||||
p._inflight = set()
|
||||
p._inflight_lock = threading.Lock()
|
||||
return p
|
||||
|
||||
|
||||
class FailureRecordingTests(unittest.TestCase):
|
||||
def run_failure(self, method: str, err: Exception) -> str:
|
||||
cur = FakeCursor(rows=[(1, "RECEIVED")])
|
||||
p = bare_pipeline()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
|
||||
getattr(p, method)("id-1", err)
|
||||
self.assertEqual(len(cur.executed), 1)
|
||||
return cur.executed[0][0]
|
||||
|
||||
def test_ingest_transient_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
|
||||
self.assertNotIn("attempts + 1", sql)
|
||||
self.assertNotIn("FAILED", sql)
|
||||
|
||||
def test_ingest_db_outage_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
|
||||
self.assertNotIn("attempts + 1", sql)
|
||||
|
||||
def test_ingest_permanent_counts_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
|
||||
self.assertIn("attempts + 1", sql)
|
||||
self.assertIn("'FAILED'", sql)
|
||||
|
||||
def test_ocr_transient_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
|
||||
self.assertNotIn("ocr_attempts + 1", sql)
|
||||
self.assertNotIn("'FAIL'", sql)
|
||||
|
||||
def test_ocr_permanent_counts_attempt(self):
|
||||
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
|
||||
self.assertIn("ocr_attempts + 1", sql)
|
||||
self.assertIn("'FAIL'", sql)
|
||||
|
||||
|
||||
class SubmitDedupeTests(unittest.TestCase):
|
||||
def test_same_id_is_queued_once_until_done(self):
|
||||
p = bare_pipeline()
|
||||
pool = mock.Mock()
|
||||
calls = []
|
||||
p.ingest = lambda arg: calls.append(arg)
|
||||
p.ingest.__name__ = "ingest"
|
||||
|
||||
p._submit(pool, p.ingest, "a")
|
||||
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
|
||||
p._submit(pool, p.ingest, "b")
|
||||
self.assertEqual(pool.submit.call_count, 2)
|
||||
|
||||
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
|
||||
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
|
||||
p._guarded(token, fn, arg)
|
||||
self.assertEqual(calls, ["a"])
|
||||
p._submit(pool, p.ingest, "a")
|
||||
self.assertEqual(pool.submit.call_count, 3)
|
||||
|
||||
def test_failing_worker_releases_the_id(self):
|
||||
p = bare_pipeline()
|
||||
pool = mock.Mock()
|
||||
|
||||
def boom(arg):
|
||||
raise RuntimeError("x")
|
||||
|
||||
p._submit(pool, boom, "a")
|
||||
token, fn, arg = pool.submit.call_args.args[1:4]
|
||||
with mock.patch("builtins.print"):
|
||||
p._guarded(token, fn, arg)
|
||||
self.assertEqual(p._inflight, set())
|
||||
|
||||
|
||||
class ReceiverStatusCodeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
|
||||
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
|
||||
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
|
||||
cls.port = cls.server.server_address[1]
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
||||
cls.thread.start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.shutdown()
|
||||
cls.server.server_close()
|
||||
|
||||
def post(self, path, body, content_type):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
try:
|
||||
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
|
||||
resp = conn.getresponse()
|
||||
return resp.status, resp.getheader("Retry-After"), resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def attachment(self):
|
||||
boundary = "xx"
|
||||
body = (
|
||||
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
|
||||
"Content-Type: image/jpeg\r\n\r\n"
|
||||
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
|
||||
path = f"/v2/events/{uuid.uuid4()}/attachments/"
|
||||
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
|
||||
|
||||
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
|
||||
status, retry_after, _ = self.attachment()
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||
|
||||
def test_unexpected_error_on_upload_stays_500(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
|
||||
status, retry_after, _ = self.attachment()
|
||||
self.assertEqual(status, 500)
|
||||
self.assertIsNone(retry_after)
|
||||
|
||||
def test_db_outage_on_event_gives_503(self):
|
||||
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
|
||||
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
|
||||
mock.patch("builtins.print"):
|
||||
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||
|
||||
def test_unknown_object_id_is_still_404(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
|
||||
status, _, _ = self.attachment()
|
||||
self.assertEqual(status, 404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user