"""Backoff und Fehlerbehandlung des Legacy-Pollers mit simulierten S3-/DB-Stoerungen. python -m unittest discover -s tests -v (oder: python -m pytest tests) Braucht die Pakete aus requirements.txt, aber weder Netz noch DB. """ import contextlib import io import json import os import sys import tempfile import unittest from datetime import datetime, timedelta, timezone from unittest import mock sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import psycopg # noqa: E402 from botocore.exceptions import ( # noqa: E402 ClientError, ConnectTimeoutError, ConnectionClosedError, EndpointConnectionError, ReadTimeoutError, ) import main # noqa: E402 def client_error(code: str, http: int, op: str = "ListObjectsV2") -> ClientError: return ClientError({"Error": {"Code": code, "Message": code}, "ResponseMetadata": {"HTTPStatusCode": http}}, op) def s3_unavailable() -> ClientError: return client_error("ServiceUnavailable", 503) def make_cfg(tmpdir: str, **overrides) -> main.AppConfig: params = dict( s3_endpoint="https://s3.invalid/", s3_access_key="test", s3_secret_key="test", s3_bucket="bucket", pg_dsn="postgresql://invalid/db", poll_seconds=30, parallel_workers=1, backoff_initial_seconds=5, backoff_max_seconds=300, heartbeat_seconds=300, heartbeat_file=os.path.join(tmpdir, "heartbeat.json"), ) params.update(overrides) return main.AppConfig(**params) class FakeClock: """Zeit laeuft nur durch sleep() - Tests laufen ohne echtes Warten.""" def __init__(self, start: float = 1_000_000.0): self.now = start self.sleeps: list[float] = [] def time(self) -> float: return self.now def sleep(self, seconds: float): self.sleeps.append(seconds) self.now += seconds class ScriptedCycle: """Zyklus-Ersatz: arbeitet eine Liste ab (Exception -> werfen, int -> zurueckgeben).""" def __init__(self, steps): self.steps = list(steps) self.calls = 0 def __call__(self, cfg, stats): self.calls += 1 step = self.steps.pop(0) if self.steps else 0 if isinstance(step, BaseException): raise step if callable(step): return step(stats) return step class FakeS3: """Simulierter S3-Client: Listing liefert Objekte oder wirft den angegebenen Fehler.""" def __init__(self, objects=None, list_error: BaseException = None): self.objects = objects or [] self.list_error = list_error def list_entrance_objects(self): if self.list_error is not None: raise self.list_error yield from self.objects class FakeConn: """Simulierte psycopg-Verbindung; die SQL-Funktionen werden in den Tests gepatcht.""" def __enter__(self): return self def __exit__(self, *exc): return False @contextlib.contextmanager def transaction(self): yield def cursor(self): return contextlib.nullcontext(object()) def entrance_objects(*uuids): old = datetime.now(timezone.utc) - timedelta(hours=1) objs = [] for u in uuids: objs.append({"Key": f"icu/entrance/{u}.jpg", "LastModified": old}) objs.append({"Key": f"icu/entrance/{u}.json", "LastModified": old}) return objs UUIDS = [f"00000000-0000-4000-8000-00000000000{i}" for i in range(1, 4)] class BackoffTest(unittest.TestCase): def test_doubles_up_to_maximum(self): b = main.Backoff(5, 300) self.assertEqual([b.next_delay() for _ in range(9)], [5, 10, 20, 40, 80, 160, 300, 300, 300]) def test_reset_starts_again_at_initial(self): b = main.Backoff(5, 300) for _ in range(4): b.next_delay() b.reset() self.assertEqual(b.failures, 0) self.assertEqual(b.next_delay(), 5) def test_long_outage_does_not_overflow(self): b = main.Backoff(5, 300) for _ in range(5000): delay = b.next_delay() self.assertEqual(delay, 300) class ErrorClassificationTest(unittest.TestCase): def test_transient(self): cases = [ s3_unavailable(), client_error("SlowDown", 503), client_error("InternalError", 500), client_error("TooManyRequests", 429), client_error("Whatever", 502), EndpointConnectionError(endpoint_url="https://s3.invalid"), ConnectTimeoutError(endpoint_url="https://s3.invalid"), ReadTimeoutError(endpoint_url="https://s3.invalid"), ConnectionClosedError(endpoint_url="https://s3.invalid"), psycopg.OperationalError("connection refused"), psycopg.errors.AdminShutdown("terminating connection due to administrator command"), ConnectionResetError(), TimeoutError(), ] for exc in cases: with self.subTest(exc=repr(exc)): self.assertTrue(main.is_transient_error(exc)) def test_unexpected(self): cases = [ client_error("AccessDenied", 403), client_error("NoSuchKey", 404, "GetObject"), client_error("NoSuchBucket", 404), psycopg.ProgrammingError("syntax error"), psycopg.errors.InsufficientPrivilege("permission denied"), KeyError("status"), ValueError("metadata.servertime missing"), ] for exc in cases: with self.subTest(exc=repr(exc)): self.assertFalse(main.is_transient_error(exc)) def test_follows_explicit_cause_only(self): try: try: raise s3_unavailable() except ClientError as e: raise RuntimeError("wrapped") from e except RuntimeError as wrapped: self.assertTrue(main.is_transient_error(wrapped)) # Ein Bug im Fehlerpfad (nur __context__) bleibt ein unerwarteter Fehler. try: try: raise s3_unavailable() except ClientError: raise KeyError("bug in handler") except KeyError as bug: self.assertFalse(main.is_transient_error(bug)) class PollForeverTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.cfg = make_cfg(self.tmp.name) self.clock = FakeClock() self.out = io.StringIO() self.err = io.StringIO() def tearDown(self): self.tmp.cleanup() def run_loop(self, cycle, iterations, init_schema=lambda cfg: None): with contextlib.redirect_stdout(self.out), contextlib.redirect_stderr(self.err): return main.poll_forever( self.cfg, cycle=cycle, init_schema=init_schema, sleep=self.clock.sleep, clock=self.clock.time, max_iterations=iterations, ) def test_s3_outage_backs_off_and_recovers(self): cycle = ScriptedCycle([s3_unavailable(), s3_unavailable(), s3_unavailable(), 0, 0, s3_unavailable(), 0]) stats = self.run_loop(cycle, 7) self.assertEqual(self.clock.sleeps, [5, 10, 20, 30, 30, 5, 30]) self.assertEqual(stats.cycles, 7) self.assertEqual(stats.cycles_failed, 4) self.assertEqual(stats.errors_transient, 4) self.assertEqual(stats.errors_other, 0) self.assertEqual(stats.consecutive_failures, 0) self.assertIsNotNone(stats.last_success) log = self.out.getvalue() self.assertIn("[CYCLE_FAIL] kind=transient consecutive=1 retry_in=5s error=ClientError", log) self.assertIn("[RECOVERED] after_failures=3 outage_s=35", log) self.assertNotIn("Traceback", self.err.getvalue()) def test_backoff_is_capped_during_long_outage(self): stats = self.run_loop(ScriptedCycle([s3_unavailable()] * 12), 12) self.assertEqual(self.clock.sleeps[-4:], [300, 300, 300, 300]) self.assertEqual(stats.consecutive_failures, 12) def test_db_unavailable_at_start_is_retried_then_schema_init_runs_once(self): calls = [] def init_schema(cfg): calls.append(1) if len(calls) <= 2: raise psycopg.OperationalError("could not connect to server") cycle = ScriptedCycle([0, 0, 0]) stats = self.run_loop(cycle, 5, init_schema=init_schema) self.assertEqual(len(calls), 3) self.assertEqual(cycle.calls, 3) self.assertEqual(self.clock.sleeps, [5, 10, 30, 30, 30]) self.assertEqual(stats.errors_transient, 2) def test_unexpected_error_is_logged_with_traceback_but_does_not_crash(self): cycle = ScriptedCycle([KeyError("boom"), KeyError("boom"), 0]) stats = self.run_loop(cycle, 3) self.assertEqual(stats.errors_other, 2) self.assertEqual(self.clock.sleeps, [5, 10, 30]) self.assertIn("[CYCLE_FAIL] kind=unexpected", self.out.getvalue()) self.assertIn("Traceback", self.err.getvalue()) self.assertIn("KeyError: 'boom'", self.err.getvalue()) def test_no_busy_loop_when_ready_sets_keep_failing(self): def progress(stats): stats.image_done(True) return 1 cycle = ScriptedCycle([2, progress, 0]) self.run_loop(cycle, 3) # 2 Saetze ohne Erfolg -> Poll-Pause; Fortschritt -> sofort weiter; leer -> Poll-Pause self.assertEqual(self.clock.sleeps, [30, 30]) def test_heartbeat_log_and_file(self): stats = self.run_loop(ScriptedCycle([0] * 12), 12) beats = [line for line in self.out.getvalue().splitlines() if line.startswith("[HEARTBEAT]")] self.assertEqual(len(beats), 1) # Zyklus 11 laeuft bei +300 s -> genau eine Zeile self.assertIn("cycles=11 cycles_failed=0", beats[0]) self.assertIn("last_success_age=0s", beats[0]) with open(self.cfg.heartbeat_file, encoding="utf-8") as f: data = json.load(f) self.assertEqual(data["cycles"], stats.cycles) self.assertEqual(data["errors_transient"], 0) class RunOnceWithFakeS3Test(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.cfg = make_cfg(self.tmp.name) def tearDown(self): self.tmp.cleanup() def test_listing_failure_propagates_to_loop_which_survives(self): s3 = FakeS3(list_error=s3_unavailable()) with self.assertRaises(ClientError): main.run_once(self.cfg, None, s3c=s3) clock = FakeClock() with contextlib.redirect_stdout(io.StringIO()): stats = main.poll_forever( self.cfg, cycle=lambda cfg, st: main.run_once(cfg, st, s3c=s3), init_schema=lambda cfg: None, sleep=clock.sleep, clock=clock.time, max_iterations=3, ) self.assertEqual(clock.sleeps, [5, 10, 20]) self.assertEqual(stats.errors_transient, 3) def test_empty_entrance_returns_zero(self): self.assertEqual(main.run_once(self.cfg, None, s3c=FakeS3()), 0) class ProcessBatchTest(unittest.TestCase): """Transaktion pro Bild bleibt; bei S3-/DB-Stoerung wird der Rest des Batches nicht angefasst.""" def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.cfg = make_cfg(self.tmp.name) self.status_calls = [] self.processed = [] patches = [ mock.patch.object(main, "pg_connect", lambda cfg: FakeConn()), mock.patch.object(main, "is_completed_job_materialized", lambda cur, s3, cfg, uuid: False), mock.patch.object(main, "job_upsert_new", lambda cur, uuid, *a: None), mock.patch.object(main, "job_get", lambda cur, uuid: {"status": "CLEANED"}), mock.patch.object( main, "job_set_status", lambda cur, uuid, status, **kw: self.status_calls.append((uuid, status, kw.get("err"))), ), ] for p in patches: p.start() self.addCleanup(p.stop) def tearDown(self): self.tmp.cleanup() def batch(self): s3 = FakeS3(entrance_objects(*UUIDS)) return main.collect_ready_sets(s3, self.cfg), s3 def fail_first_with(self, exc): def process_one(cur, s3, cfg, s): self.processed.append(s.uuid) if s.uuid == UUIDS[0]: raise exc return mock.patch.object(main, "process_one", process_one) def test_transient_s3_error_marks_image_error_and_aborts_batch(self): sets, s3 = self.batch() stats = main.PollerStats(None) with self.fail_first_with(client_error("ServiceUnavailable", 503, "GetObject")), \ contextlib.redirect_stdout(io.StringIO()) as out: with self.assertRaises(ClientError): main._process_ready_batch(self.cfg, sets, s3, stats) self.assertEqual(self.processed, [UUIDS[0]]) self.assertEqual([(u, st) for u, st, _ in self.status_calls], [(UUIDS[0], "ERROR")]) self.assertIn(f"[RESULT] uuid={UUIDS[0]} result=failure status=ERROR", out.getvalue()) self.assertEqual((stats.images_ok, stats.images_failed), (0, 1)) def test_bad_image_does_not_stop_batch(self): sets, s3 = self.batch() stats = main.PollerStats(None) with self.fail_first_with(ValueError("metadata.servertime missing")), \ contextlib.redirect_stdout(io.StringIO()): main._process_ready_batch(self.cfg, sets, s3, stats) self.assertEqual(self.processed, UUIDS) self.assertEqual(self.status_calls, [(UUIDS[0], "ERROR", "metadata.servertime missing")]) self.assertEqual((stats.images_ok, stats.images_failed), (2, 1)) def test_db_down_propagates_as_transient(self): sets, s3 = self.batch() def no_db(cfg): raise psycopg.OperationalError("connection timeout expired") with mock.patch.object(main, "pg_connect", no_db): with self.assertRaises(psycopg.OperationalError) as ctx: main._process_ready_batch(self.cfg, sets, s3) self.assertTrue(main.is_transient_error(ctx.exception)) def test_run_once_parallel_raises_when_a_worker_hits_outage(self): cfg = make_cfg(self.tmp.name, parallel_workers=3) s3 = FakeS3(entrance_objects(*UUIDS)) with self.fail_first_with(s3_unavailable()), contextlib.redirect_stdout(io.StringIO()): with self.assertRaises(ClientError): main.run_once(cfg, main.PollerStats(None), s3c=s3) self.assertEqual(sorted(self.processed), UUIDS) # die anderen Worker laufen zu Ende class S3ClientConfigTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.cfg = make_cfg(self.tmp.name) def tearDown(self): self.tmp.cleanup() def test_poller_client_has_explicit_timeouts_and_retries(self): conf = main.S3(self.cfg, main.s3_client_config(self.cfg)).client.meta.config self.assertEqual(conf.connect_timeout, 10) self.assertEqual(conf.read_timeout, 30) self.assertEqual(conf.retries.get("mode"), "standard") self.assertEqual(conf.retries.get("total_max_attempts"), 4) self.assertNotIn("max_attempts", conf.retries) self.assertEqual(conf.signature_version, "s3v4") self.assertEqual(conf.s3.get("addressing_style"), "path") def test_default_client_unchanged_for_gundi_path(self): import boto3 from botocore.config import Config # exakt wie vor der Aenderung gebaut before = boto3.client( "s3", endpoint_url=self.cfg.s3_endpoint, aws_access_key_id=self.cfg.s3_access_key, aws_secret_access_key=self.cfg.s3_secret_key, config=Config(max_pool_connections=self.cfg.s3_max_pool_connections), ).meta.config now = main.S3(self.cfg).client.meta.config for attr in ("connect_timeout", "read_timeout", "retries", "max_pool_connections", "signature_version", "s3", "region_name"): with self.subTest(attr=attr): self.assertEqual(getattr(now, attr), getattr(before, attr)) self.assertEqual(now.retries, {"mode": "legacy"}) if __name__ == "__main__": unittest.main()