"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck. python -m unittest discover -s tests -v (oder: python -m pytest tests) Braucht die Pakete aus requirements.txt, aber weder Netz noch DB. """ import http.client import json import os import sys import threading import unittest import uuid from types import SimpleNamespace from unittest import mock sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import psycopg # noqa: E402 from botocore.exceptions import ClientError # noqa: E402 import gundi_pipeline as gp # noqa: E402 import gundi_receiver as gr # noqa: E402 def s3_unavailable() -> ClientError: return ClientError( {"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject", ) class FakeCursor: def __init__(self, rows=None, rowcount=1): self.executed: list[tuple[str, tuple]] = [] self._rows = list(rows or []) self.rowcount = rowcount def __enter__(self): return self def __exit__(self, *exc): return False def execute(self, sql, params=()): self.executed.append((" ".join(sql.split()), params)) def fetchone(self): return self._rows.pop(0) if self._rows else None class FakeConn: def __init__(self, cursor: FakeCursor): self._cursor = cursor def __enter__(self): return self def __exit__(self, *exc): return False def cursor(self): return self._cursor def fake_connect(cursor: FakeCursor): return lambda dsn: FakeConn(cursor) GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3) class AttachImageTests(unittest.TestCase): def test_s3_down_raises_without_db_lock_and_without_update(self): cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")]) s3 = mock.Mock() s3.put_bytes.side_effect = s3_unavailable() with mock.patch.object(gp, "connect", fake_connect(cur)): with self.assertRaises(ClientError): gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff") sqls = [sql for sql, _ in cur.executed] self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend {"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject"))) def test_success_stores_then_updates(self): cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")]) s3 = mock.Mock() with mock.patch.object(gp, "connect", fake_connect(cur)): self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored") s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg") self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0]) def test_already_received_is_duplicate_without_s3(self): cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")]) s3 = mock.Mock() with mock.patch.object(gp, "connect", fake_connect(cur)): self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate") s3.put_bytes.assert_not_called() def test_parallel_upload_lost_the_race_is_duplicate(self): cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0) with mock.patch.object(gp, "connect", fake_connect(cur)): self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate") def test_unknown_or_wrong_type_is_lookup_error(self): for rows in ([], [("camera_status_report", "INGESTED")]): cur = FakeCursor(rows=rows) with mock.patch.object(gp, "connect", fake_connect(cur)): with self.assertRaises(LookupError): gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg") def bare_pipeline() -> gp.Pipeline: p = gp.Pipeline.__new__(gp.Pipeline) p.cfg = SimpleNamespace(pg_dsn="dsn") p.gcfg = GCFG p._inflight = set() p._inflight_lock = threading.Lock() return p class FailureRecordingTests(unittest.TestCase): def run_failure(self, method: str, err: Exception) -> str: cur = FakeCursor(rows=[(1, "RECEIVED")]) p = bare_pipeline() with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"): getattr(p, method)("id-1", err) self.assertEqual(len(cur.executed), 1) return cur.executed[0][0] def test_ingest_transient_does_not_count_attempt(self): sql = self.run_failure("_record_ingest_failure", s3_unavailable()) self.assertNotIn("attempts + 1", sql) self.assertNotIn("FAILED", sql) def test_ingest_db_outage_does_not_count_attempt(self): sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired")) self.assertNotIn("attempts + 1", sql) def test_ingest_permanent_counts_attempt(self): sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild")) self.assertIn("attempts + 1", sql) self.assertIn("'FAILED'", sql) def test_ocr_transient_does_not_count_attempt(self): sql = self.run_failure("_record_ocr_failure", s3_unavailable()) self.assertNotIn("ocr_attempts + 1", sql) self.assertNotIn("'FAIL'", sql) def test_ocr_permanent_counts_attempt(self): sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel")) self.assertIn("ocr_attempts + 1", sql) self.assertIn("'FAIL'", sql) class BacklogAlarmTests(unittest.TestCase): def run_check(self, inbox_row, image_row, pipeline=None): cur = FakeCursor(rows=[inbox_row, image_row]) p = pipeline or bare_pipeline() p.gcfg = SimpleNamespace(alarm_after_seconds=1800) p._last_alarm = 0.0 out = [] with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))): p.check_backlog() return p, cur, out def test_silent_when_nothing_is_wrong(self): _, _, out = self.run_check((0, 0, 0), (0, 0)) self.assertEqual(out, []) def test_alarm_for_stuck_and_new_failures(self): _, _, out = self.run_check((2, 0, 1), (0, 3)) self.assertEqual(len(out), 1) self.assertIn("[GUNDI][ALARM]", out[0]) self.assertIn("received_haengt=2", out[0]) self.assertIn("failed_24h=1", out[0]) self.assertIn("ocr_fail_24h=3", out[0]) def test_failures_older_than_24h_are_not_counted(self): _, cur, _ = self.run_check((0, 0, 0), (0, 0)) sqls = " ".join(sql for sql, _ in cur.executed) self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls) self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls) def test_alarm_is_throttled(self): p, _, first = self.run_check((1, 0, 0), (0, 0)) self.assertEqual(len(first), 1) cur = FakeCursor(rows=[(1, 0, 0), (0, 0)]) out = [] with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)): p.check_backlog() # direkt danach -> unterdrueckt self.assertEqual(out, []) class SubmitDedupeTests(unittest.TestCase): def test_same_id_is_queued_once_until_done(self): p = bare_pipeline() pool = mock.Mock() calls = [] p.ingest = lambda arg: calls.append(arg) p.ingest.__name__ = "ingest" p._submit(pool, p.ingest, "a") p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert p._submit(pool, p.ingest, "b") self.assertEqual(pool.submit.call_count, 2) # Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden token, fn, arg = pool.submit.call_args_list[0].args[1:4] p._guarded(token, fn, arg) self.assertEqual(calls, ["a"]) p._submit(pool, p.ingest, "a") self.assertEqual(pool.submit.call_count, 3) def test_failing_worker_releases_the_id(self): p = bare_pipeline() pool = mock.Mock() def boom(arg): raise RuntimeError("x") p._submit(pool, boom, "a") token, fn, arg = pool.submit.call_args.args[1:4] with mock.patch("builtins.print"): p._guarded(token, fn, arg) self.assertEqual(p._inflight, set()) class ReceiverStatusCodeTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20) cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock()) cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline) cls.port = cls.server.server_address[1] cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) cls.thread.start() @classmethod def tearDownClass(cls): cls.server.shutdown() cls.server.server_close() def post(self, path, body, content_type): conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) try: conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key}) resp = conn.getresponse() return resp.status, resp.getheader("Retry-After"), resp.read() finally: conn.close() def attachment(self): boundary = "xx" body = ( f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n' "Content-Type: image/jpeg\r\n\r\n" ).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode() path = f"/v2/events/{uuid.uuid4()}/attachments/" return self.post(path, body, f"multipart/form-data; boundary={boundary}") def test_s3_outage_on_upload_gives_503_with_retry_after(self): with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"): status, retry_after, _ = self.attachment() self.assertEqual(status, 503) self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS)) def test_unexpected_error_on_upload_stays_500(self): with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"): status, retry_after, _ = self.attachment() self.assertEqual(status, 500) self.assertIsNone(retry_after) def test_db_outage_on_event_gives_503(self): event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"} with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \ mock.patch("builtins.print"): status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json") self.assertEqual(status, 503) self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS)) def test_unknown_object_id_is_still_404(self): with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")): status, _, _ = self.attachment() self.assertEqual(status, 404) if __name__ == "__main__": unittest.main()