- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
253 lines
9.8 KiB
Python
253 lines
9.8 KiB
Python
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
|
|
|
|
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
|
|
|
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
|
"""
|
|
import http.client
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
import unittest
|
|
import uuid
|
|
from types import SimpleNamespace
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import psycopg # noqa: E402
|
|
from botocore.exceptions import ClientError # noqa: E402
|
|
|
|
import gundi_pipeline as gp # noqa: E402
|
|
import gundi_receiver as gr # noqa: E402
|
|
|
|
|
|
def s3_unavailable() -> ClientError:
|
|
return ClientError(
|
|
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
|
|
"PutObject",
|
|
)
|
|
|
|
|
|
class FakeCursor:
|
|
def __init__(self, rows=None, rowcount=1):
|
|
self.executed: list[tuple[str, tuple]] = []
|
|
self._rows = list(rows or [])
|
|
self.rowcount = rowcount
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def execute(self, sql, params=()):
|
|
self.executed.append((" ".join(sql.split()), params))
|
|
|
|
def fetchone(self):
|
|
return self._rows.pop(0) if self._rows else None
|
|
|
|
|
|
class FakeConn:
|
|
def __init__(self, cursor: FakeCursor):
|
|
self._cursor = cursor
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def cursor(self):
|
|
return self._cursor
|
|
|
|
|
|
def fake_connect(cursor: FakeCursor):
|
|
return lambda dsn: FakeConn(cursor)
|
|
|
|
|
|
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
|
|
|
|
|
|
class AttachImageTests(unittest.TestCase):
|
|
def test_s3_down_raises_without_db_lock_and_without_update(self):
|
|
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
|
|
s3 = mock.Mock()
|
|
s3.put_bytes.side_effect = s3_unavailable()
|
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
|
with self.assertRaises(ClientError):
|
|
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
|
|
sqls = [sql for sql, _ in cur.executed]
|
|
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
|
|
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
|
|
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
|
|
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
|
|
|
|
def test_success_stores_then_updates(self):
|
|
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
|
|
s3 = mock.Mock()
|
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
|
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
|
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
|
|
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
|
|
|
def test_already_received_is_duplicate_without_s3(self):
|
|
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
|
|
s3 = mock.Mock()
|
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
|
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
|
|
s3.put_bytes.assert_not_called()
|
|
|
|
def test_parallel_upload_lost_the_race_is_duplicate(self):
|
|
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
|
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
|
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
|
|
|
|
def test_unknown_or_wrong_type_is_lookup_error(self):
|
|
for rows in ([], [("camera_status_report", "INGESTED")]):
|
|
cur = FakeCursor(rows=rows)
|
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
|
with self.assertRaises(LookupError):
|
|
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
|
|
|
|
|
|
def bare_pipeline() -> gp.Pipeline:
|
|
p = gp.Pipeline.__new__(gp.Pipeline)
|
|
p.cfg = SimpleNamespace(pg_dsn="dsn")
|
|
p.gcfg = GCFG
|
|
p._inflight = set()
|
|
p._inflight_lock = threading.Lock()
|
|
return p
|
|
|
|
|
|
class FailureRecordingTests(unittest.TestCase):
|
|
def run_failure(self, method: str, err: Exception) -> str:
|
|
cur = FakeCursor(rows=[(1, "RECEIVED")])
|
|
p = bare_pipeline()
|
|
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
|
|
getattr(p, method)("id-1", err)
|
|
self.assertEqual(len(cur.executed), 1)
|
|
return cur.executed[0][0]
|
|
|
|
def test_ingest_transient_does_not_count_attempt(self):
|
|
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
|
|
self.assertNotIn("attempts + 1", sql)
|
|
self.assertNotIn("FAILED", sql)
|
|
|
|
def test_ingest_db_outage_does_not_count_attempt(self):
|
|
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
|
|
self.assertNotIn("attempts + 1", sql)
|
|
|
|
def test_ingest_permanent_counts_attempt(self):
|
|
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
|
|
self.assertIn("attempts + 1", sql)
|
|
self.assertIn("'FAILED'", sql)
|
|
|
|
def test_ocr_transient_does_not_count_attempt(self):
|
|
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
|
|
self.assertNotIn("ocr_attempts + 1", sql)
|
|
self.assertNotIn("'FAIL'", sql)
|
|
|
|
def test_ocr_permanent_counts_attempt(self):
|
|
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
|
|
self.assertIn("ocr_attempts + 1", sql)
|
|
self.assertIn("'FAIL'", sql)
|
|
|
|
|
|
class SubmitDedupeTests(unittest.TestCase):
|
|
def test_same_id_is_queued_once_until_done(self):
|
|
p = bare_pipeline()
|
|
pool = mock.Mock()
|
|
calls = []
|
|
p.ingest = lambda arg: calls.append(arg)
|
|
p.ingest.__name__ = "ingest"
|
|
|
|
p._submit(pool, p.ingest, "a")
|
|
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
|
|
p._submit(pool, p.ingest, "b")
|
|
self.assertEqual(pool.submit.call_count, 2)
|
|
|
|
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
|
|
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
|
|
p._guarded(token, fn, arg)
|
|
self.assertEqual(calls, ["a"])
|
|
p._submit(pool, p.ingest, "a")
|
|
self.assertEqual(pool.submit.call_count, 3)
|
|
|
|
def test_failing_worker_releases_the_id(self):
|
|
p = bare_pipeline()
|
|
pool = mock.Mock()
|
|
|
|
def boom(arg):
|
|
raise RuntimeError("x")
|
|
|
|
p._submit(pool, boom, "a")
|
|
token, fn, arg = pool.submit.call_args.args[1:4]
|
|
with mock.patch("builtins.print"):
|
|
p._guarded(token, fn, arg)
|
|
self.assertEqual(p._inflight, set())
|
|
|
|
|
|
class ReceiverStatusCodeTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
|
|
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
|
|
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
|
|
cls.port = cls.server.server_address[1]
|
|
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
|
cls.thread.start()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.server.shutdown()
|
|
cls.server.server_close()
|
|
|
|
def post(self, path, body, content_type):
|
|
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
|
try:
|
|
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
|
|
resp = conn.getresponse()
|
|
return resp.status, resp.getheader("Retry-After"), resp.read()
|
|
finally:
|
|
conn.close()
|
|
|
|
def attachment(self):
|
|
boundary = "xx"
|
|
body = (
|
|
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
|
|
"Content-Type: image/jpeg\r\n\r\n"
|
|
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
|
|
path = f"/v2/events/{uuid.uuid4()}/attachments/"
|
|
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
|
|
|
|
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
|
|
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
|
|
status, retry_after, _ = self.attachment()
|
|
self.assertEqual(status, 503)
|
|
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
|
|
|
def test_unexpected_error_on_upload_stays_500(self):
|
|
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
|
|
status, retry_after, _ = self.attachment()
|
|
self.assertEqual(status, 500)
|
|
self.assertIsNone(retry_after)
|
|
|
|
def test_db_outage_on_event_gives_503(self):
|
|
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
|
|
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
|
|
mock.patch("builtins.print"):
|
|
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
|
|
self.assertEqual(status, 503)
|
|
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
|
|
|
def test_unknown_object_id_is_still_404(self):
|
|
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
|
|
status, _, _ = self.attachment()
|
|
self.assertEqual(status, 404)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|