Files
melesICUmover/tests/test_gundi_resilience.py
domandClaude Opus 5.5 ed105fbff8 Prepare legacy migration for the portal cutover
- detection from legacy category only for scores >= 0.5, without ''/none
- category_label + view detection_labeled unify ICU and legacy labels
- schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images
  (same imei, same size, <= 10 min), archived in image_superseded
- migrate_legacy --plan: read-only preview of the migration
- Gundi S3 prefix icu/gundi-test/ -> icu/gundi/

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 21:01:17 +02:00

292 lines
12 KiB
Python

"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
python -m unittest discover -s tests -v (oder: python -m pytest tests)
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
"""
import http.client
import json
import os
import sys
import threading
import unittest
import uuid
from types import SimpleNamespace
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import psycopg # noqa: E402
from botocore.exceptions import ClientError # noqa: E402
import gundi_pipeline as gp # noqa: E402
import gundi_receiver as gr # noqa: E402
def s3_unavailable() -> ClientError:
return ClientError(
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
"PutObject",
)
class FakeCursor:
def __init__(self, rows=None, rowcount=1):
self.executed: list[tuple[str, tuple]] = []
self._rows = list(rows or [])
self.rowcount = rowcount
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def execute(self, sql, params=()):
self.executed.append((" ".join(sql.split()), params))
def fetchone(self):
return self._rows.pop(0) if self._rows else None
class FakeConn:
def __init__(self, cursor: FakeCursor):
self._cursor = cursor
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def cursor(self):
return self._cursor
def fake_connect(cursor: FakeCursor):
return lambda dsn: FakeConn(cursor)
GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3)
class AttachImageTests(unittest.TestCase):
def test_s3_down_raises_without_db_lock_and_without_update(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
s3 = mock.Mock()
s3.put_bytes.side_effect = s3_unavailable()
with mock.patch.object(gp, "connect", fake_connect(cur)):
with self.assertRaises(ClientError):
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
sqls = [sql for sql, _ in cur.executed]
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
def test_success_stores_then_updates(self):
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg")
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
def test_already_received_is_duplicate_without_s3(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
s3.put_bytes.assert_not_called()
def test_parallel_upload_lost_the_race_is_duplicate(self):
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
def test_unknown_or_wrong_type_is_lookup_error(self):
for rows in ([], [("camera_status_report", "INGESTED")]):
cur = FakeCursor(rows=rows)
with mock.patch.object(gp, "connect", fake_connect(cur)):
with self.assertRaises(LookupError):
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
def bare_pipeline() -> gp.Pipeline:
p = gp.Pipeline.__new__(gp.Pipeline)
p.cfg = SimpleNamespace(pg_dsn="dsn")
p.gcfg = GCFG
p._inflight = set()
p._inflight_lock = threading.Lock()
return p
class FailureRecordingTests(unittest.TestCase):
def run_failure(self, method: str, err: Exception) -> str:
cur = FakeCursor(rows=[(1, "RECEIVED")])
p = bare_pipeline()
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
getattr(p, method)("id-1", err)
self.assertEqual(len(cur.executed), 1)
return cur.executed[0][0]
def test_ingest_transient_does_not_count_attempt(self):
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
self.assertNotIn("attempts + 1", sql)
self.assertNotIn("FAILED", sql)
def test_ingest_db_outage_does_not_count_attempt(self):
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
self.assertNotIn("attempts + 1", sql)
def test_ingest_permanent_counts_attempt(self):
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
self.assertIn("attempts + 1", sql)
self.assertIn("'FAILED'", sql)
def test_ocr_transient_does_not_count_attempt(self):
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
self.assertNotIn("ocr_attempts + 1", sql)
self.assertNotIn("'FAIL'", sql)
def test_ocr_permanent_counts_attempt(self):
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
self.assertIn("ocr_attempts + 1", sql)
self.assertIn("'FAIL'", sql)
class BacklogAlarmTests(unittest.TestCase):
def run_check(self, inbox_row, image_row, pipeline=None):
cur = FakeCursor(rows=[inbox_row, image_row])
p = pipeline or bare_pipeline()
p.gcfg = SimpleNamespace(alarm_after_seconds=1800)
p._last_alarm = 0.0
out = []
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))):
p.check_backlog()
return p, cur, out
def test_silent_when_nothing_is_wrong(self):
_, _, out = self.run_check((0, 0, 0), (0, 0))
self.assertEqual(out, [])
def test_alarm_for_stuck_and_new_failures(self):
_, _, out = self.run_check((2, 0, 1), (0, 3))
self.assertEqual(len(out), 1)
self.assertIn("[GUNDI][ALARM]", out[0])
self.assertIn("received_haengt=2", out[0])
self.assertIn("failed_24h=1", out[0])
self.assertIn("ocr_fail_24h=3", out[0])
def test_failures_older_than_24h_are_not_counted(self):
_, cur, _ = self.run_check((0, 0, 0), (0, 0))
sqls = " ".join(sql for sql, _ in cur.executed)
self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls)
self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls)
def test_alarm_is_throttled(self):
p, _, first = self.run_check((1, 0, 0), (0, 0))
self.assertEqual(len(first), 1)
cur = FakeCursor(rows=[(1, 0, 0), (0, 0)])
out = []
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)):
p.check_backlog() # direkt danach -> unterdrueckt
self.assertEqual(out, [])
class SubmitDedupeTests(unittest.TestCase):
def test_same_id_is_queued_once_until_done(self):
p = bare_pipeline()
pool = mock.Mock()
calls = []
p.ingest = lambda arg: calls.append(arg)
p.ingest.__name__ = "ingest"
p._submit(pool, p.ingest, "a")
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
p._submit(pool, p.ingest, "b")
self.assertEqual(pool.submit.call_count, 2)
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
p._guarded(token, fn, arg)
self.assertEqual(calls, ["a"])
p._submit(pool, p.ingest, "a")
self.assertEqual(pool.submit.call_count, 3)
def test_failing_worker_releases_the_id(self):
p = bare_pipeline()
pool = mock.Mock()
def boom(arg):
raise RuntimeError("x")
p._submit(pool, boom, "a")
token, fn, arg = pool.submit.call_args.args[1:4]
with mock.patch("builtins.print"):
p._guarded(token, fn, arg)
self.assertEqual(p._inflight, set())
class ReceiverStatusCodeTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
cls.port = cls.server.server_address[1]
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
cls.thread.start()
@classmethod
def tearDownClass(cls):
cls.server.shutdown()
cls.server.server_close()
def post(self, path, body, content_type):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
try:
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
resp = conn.getresponse()
return resp.status, resp.getheader("Retry-After"), resp.read()
finally:
conn.close()
def attachment(self):
boundary = "xx"
body = (
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
"Content-Type: image/jpeg\r\n\r\n"
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
path = f"/v2/events/{uuid.uuid4()}/attachments/"
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
status, retry_after, _ = self.attachment()
self.assertEqual(status, 503)
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
def test_unexpected_error_on_upload_stays_500(self):
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
status, retry_after, _ = self.attachment()
self.assertEqual(status, 500)
self.assertIsNone(retry_after)
def test_db_outage_on_event_gives_503(self):
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
mock.patch("builtins.print"):
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
self.assertEqual(status, 503)
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
def test_unknown_object_id_is_still_404(self):
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
status, _, _ = self.attachment()
self.assertEqual(status, 404)
if __name__ == "__main__":
unittest.main()