Files
melesICUmover/tests/test_poller_resilience.py
T
domandClaude Opus 5 0f574cccde Make legacy poller resilient to S3/DB outages
The poller crashed on every S3 or DB error (1162 restarts during the
fsn1 object storage degradation on 2026-09-21). Now:

- main loop catches errors per cycle, classifies them (S3 5xx/throttling,
  connection/timeouts, psycopg.OperationalError = transient; everything
  else logged with traceback) and retries with exponential backoff
  (5 s .. 5 min, reset after success); db_init runs until it succeeds once
- explicit botocore config for the poller: connect/read timeouts, standard
  retries (total attempts), s3v4, path addressing; S3(cfg) without a
  config keeps the previous client for the Gundi path
- DB connect timeout + TCP keepalives
- one S3 client per cycle shared by all workers
- a batch stops after a transient S3/DB error (image stays ERROR as before)
- no busy loop when ready sets keep failing
- [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck
- tools/entrance_inventory.py: read-only listing of icu/entrance orphans
- unit tests with simulated S3/DB failures

Per-image transaction, status names/order, keys and paths are unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-21 16:56:55 +02:00

441 lines
16 KiB
Python

"""Backoff und Fehlerbehandlung des Legacy-Pollers mit simulierten S3-/DB-Stoerungen.
python -m unittest discover -s tests -v (oder: python -m pytest tests)
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
"""
import contextlib
import io
import json
import os
import sys
import tempfile
import unittest
from datetime import datetime, timedelta, timezone
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import psycopg # noqa: E402
from botocore.exceptions import ( # noqa: E402
ClientError,
ConnectTimeoutError,
ConnectionClosedError,
EndpointConnectionError,
ReadTimeoutError,
)
import main # noqa: E402
def client_error(code: str, http: int, op: str = "ListObjectsV2") -> ClientError:
return ClientError({"Error": {"Code": code, "Message": code}, "ResponseMetadata": {"HTTPStatusCode": http}}, op)
def s3_unavailable() -> ClientError:
return client_error("ServiceUnavailable", 503)
def make_cfg(tmpdir: str, **overrides) -> main.AppConfig:
params = dict(
s3_endpoint="https://s3.invalid/",
s3_access_key="test",
s3_secret_key="test",
s3_bucket="bucket",
pg_dsn="postgresql://invalid/db",
poll_seconds=30,
parallel_workers=1,
backoff_initial_seconds=5,
backoff_max_seconds=300,
heartbeat_seconds=300,
heartbeat_file=os.path.join(tmpdir, "heartbeat.json"),
)
params.update(overrides)
return main.AppConfig(**params)
class FakeClock:
"""Zeit laeuft nur durch sleep() - Tests laufen ohne echtes Warten."""
def __init__(self, start: float = 1_000_000.0):
self.now = start
self.sleeps: list[float] = []
def time(self) -> float:
return self.now
def sleep(self, seconds: float):
self.sleeps.append(seconds)
self.now += seconds
class ScriptedCycle:
"""Zyklus-Ersatz: arbeitet eine Liste ab (Exception -> werfen, int -> zurueckgeben)."""
def __init__(self, steps):
self.steps = list(steps)
self.calls = 0
def __call__(self, cfg, stats):
self.calls += 1
step = self.steps.pop(0) if self.steps else 0
if isinstance(step, BaseException):
raise step
if callable(step):
return step(stats)
return step
class FakeS3:
"""Simulierter S3-Client: Listing liefert Objekte oder wirft den angegebenen Fehler."""
def __init__(self, objects=None, list_error: BaseException = None):
self.objects = objects or []
self.list_error = list_error
def list_entrance_objects(self):
if self.list_error is not None:
raise self.list_error
yield from self.objects
class FakeConn:
"""Simulierte psycopg-Verbindung; die SQL-Funktionen werden in den Tests gepatcht."""
def __enter__(self):
return self
def __exit__(self, *exc):
return False
@contextlib.contextmanager
def transaction(self):
yield
def cursor(self):
return contextlib.nullcontext(object())
def entrance_objects(*uuids):
old = datetime.now(timezone.utc) - timedelta(hours=1)
objs = []
for u in uuids:
objs.append({"Key": f"icu/entrance/{u}.jpg", "LastModified": old})
objs.append({"Key": f"icu/entrance/{u}.json", "LastModified": old})
return objs
UUIDS = [f"00000000-0000-4000-8000-00000000000{i}" for i in range(1, 4)]
class BackoffTest(unittest.TestCase):
def test_doubles_up_to_maximum(self):
b = main.Backoff(5, 300)
self.assertEqual([b.next_delay() for _ in range(9)], [5, 10, 20, 40, 80, 160, 300, 300, 300])
def test_reset_starts_again_at_initial(self):
b = main.Backoff(5, 300)
for _ in range(4):
b.next_delay()
b.reset()
self.assertEqual(b.failures, 0)
self.assertEqual(b.next_delay(), 5)
def test_long_outage_does_not_overflow(self):
b = main.Backoff(5, 300)
for _ in range(5000):
delay = b.next_delay()
self.assertEqual(delay, 300)
class ErrorClassificationTest(unittest.TestCase):
def test_transient(self):
cases = [
s3_unavailable(),
client_error("SlowDown", 503),
client_error("InternalError", 500),
client_error("TooManyRequests", 429),
client_error("Whatever", 502),
EndpointConnectionError(endpoint_url="https://s3.invalid"),
ConnectTimeoutError(endpoint_url="https://s3.invalid"),
ReadTimeoutError(endpoint_url="https://s3.invalid"),
ConnectionClosedError(endpoint_url="https://s3.invalid"),
psycopg.OperationalError("connection refused"),
psycopg.errors.AdminShutdown("terminating connection due to administrator command"),
ConnectionResetError(),
TimeoutError(),
]
for exc in cases:
with self.subTest(exc=repr(exc)):
self.assertTrue(main.is_transient_error(exc))
def test_unexpected(self):
cases = [
client_error("AccessDenied", 403),
client_error("NoSuchKey", 404, "GetObject"),
client_error("NoSuchBucket", 404),
psycopg.ProgrammingError("syntax error"),
psycopg.errors.InsufficientPrivilege("permission denied"),
KeyError("status"),
ValueError("metadata.servertime missing"),
]
for exc in cases:
with self.subTest(exc=repr(exc)):
self.assertFalse(main.is_transient_error(exc))
def test_follows_explicit_cause_only(self):
try:
try:
raise s3_unavailable()
except ClientError as e:
raise RuntimeError("wrapped") from e
except RuntimeError as wrapped:
self.assertTrue(main.is_transient_error(wrapped))
# Ein Bug im Fehlerpfad (nur __context__) bleibt ein unerwarteter Fehler.
try:
try:
raise s3_unavailable()
except ClientError:
raise KeyError("bug in handler")
except KeyError as bug:
self.assertFalse(main.is_transient_error(bug))
class PollForeverTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cfg = make_cfg(self.tmp.name)
self.clock = FakeClock()
self.out = io.StringIO()
self.err = io.StringIO()
def tearDown(self):
self.tmp.cleanup()
def run_loop(self, cycle, iterations, init_schema=lambda cfg: None):
with contextlib.redirect_stdout(self.out), contextlib.redirect_stderr(self.err):
return main.poll_forever(
self.cfg,
cycle=cycle,
init_schema=init_schema,
sleep=self.clock.sleep,
clock=self.clock.time,
max_iterations=iterations,
)
def test_s3_outage_backs_off_and_recovers(self):
cycle = ScriptedCycle([s3_unavailable(), s3_unavailable(), s3_unavailable(), 0, 0, s3_unavailable(), 0])
stats = self.run_loop(cycle, 7)
self.assertEqual(self.clock.sleeps, [5, 10, 20, 30, 30, 5, 30])
self.assertEqual(stats.cycles, 7)
self.assertEqual(stats.cycles_failed, 4)
self.assertEqual(stats.errors_transient, 4)
self.assertEqual(stats.errors_other, 0)
self.assertEqual(stats.consecutive_failures, 0)
self.assertIsNotNone(stats.last_success)
log = self.out.getvalue()
self.assertIn("[CYCLE_FAIL] kind=transient consecutive=1 retry_in=5s error=ClientError", log)
self.assertIn("[RECOVERED] after_failures=3 outage_s=35", log)
self.assertNotIn("Traceback", self.err.getvalue())
def test_backoff_is_capped_during_long_outage(self):
stats = self.run_loop(ScriptedCycle([s3_unavailable()] * 12), 12)
self.assertEqual(self.clock.sleeps[-4:], [300, 300, 300, 300])
self.assertEqual(stats.consecutive_failures, 12)
def test_db_unavailable_at_start_is_retried_then_schema_init_runs_once(self):
calls = []
def init_schema(cfg):
calls.append(1)
if len(calls) <= 2:
raise psycopg.OperationalError("could not connect to server")
cycle = ScriptedCycle([0, 0, 0])
stats = self.run_loop(cycle, 5, init_schema=init_schema)
self.assertEqual(len(calls), 3)
self.assertEqual(cycle.calls, 3)
self.assertEqual(self.clock.sleeps, [5, 10, 30, 30, 30])
self.assertEqual(stats.errors_transient, 2)
def test_unexpected_error_is_logged_with_traceback_but_does_not_crash(self):
cycle = ScriptedCycle([KeyError("boom"), KeyError("boom"), 0])
stats = self.run_loop(cycle, 3)
self.assertEqual(stats.errors_other, 2)
self.assertEqual(self.clock.sleeps, [5, 10, 30])
self.assertIn("[CYCLE_FAIL] kind=unexpected", self.out.getvalue())
self.assertIn("Traceback", self.err.getvalue())
self.assertIn("KeyError: 'boom'", self.err.getvalue())
def test_no_busy_loop_when_ready_sets_keep_failing(self):
def progress(stats):
stats.image_done(True)
return 1
cycle = ScriptedCycle([2, progress, 0])
self.run_loop(cycle, 3)
# 2 Saetze ohne Erfolg -> Poll-Pause; Fortschritt -> sofort weiter; leer -> Poll-Pause
self.assertEqual(self.clock.sleeps, [30, 30])
def test_heartbeat_log_and_file(self):
stats = self.run_loop(ScriptedCycle([0] * 12), 12)
beats = [line for line in self.out.getvalue().splitlines() if line.startswith("[HEARTBEAT]")]
self.assertEqual(len(beats), 1) # Zyklus 11 laeuft bei +300 s -> genau eine Zeile
self.assertIn("cycles=11 cycles_failed=0", beats[0])
self.assertIn("last_success_age=0s", beats[0])
with open(self.cfg.heartbeat_file, encoding="utf-8") as f:
data = json.load(f)
self.assertEqual(data["cycles"], stats.cycles)
self.assertEqual(data["errors_transient"], 0)
class RunOnceWithFakeS3Test(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cfg = make_cfg(self.tmp.name)
def tearDown(self):
self.tmp.cleanup()
def test_listing_failure_propagates_to_loop_which_survives(self):
s3 = FakeS3(list_error=s3_unavailable())
with self.assertRaises(ClientError):
main.run_once(self.cfg, None, s3c=s3)
clock = FakeClock()
with contextlib.redirect_stdout(io.StringIO()):
stats = main.poll_forever(
self.cfg,
cycle=lambda cfg, st: main.run_once(cfg, st, s3c=s3),
init_schema=lambda cfg: None,
sleep=clock.sleep,
clock=clock.time,
max_iterations=3,
)
self.assertEqual(clock.sleeps, [5, 10, 20])
self.assertEqual(stats.errors_transient, 3)
def test_empty_entrance_returns_zero(self):
self.assertEqual(main.run_once(self.cfg, None, s3c=FakeS3()), 0)
class ProcessBatchTest(unittest.TestCase):
"""Transaktion pro Bild bleibt; bei S3-/DB-Stoerung wird der Rest des Batches nicht angefasst."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cfg = make_cfg(self.tmp.name)
self.status_calls = []
self.processed = []
patches = [
mock.patch.object(main, "pg_connect", lambda cfg: FakeConn()),
mock.patch.object(main, "is_completed_job_materialized", lambda cur, s3, cfg, uuid: False),
mock.patch.object(main, "job_upsert_new", lambda cur, uuid, *a: None),
mock.patch.object(main, "job_get", lambda cur, uuid: {"status": "CLEANED"}),
mock.patch.object(
main, "job_set_status",
lambda cur, uuid, status, **kw: self.status_calls.append((uuid, status, kw.get("err"))),
),
]
for p in patches:
p.start()
self.addCleanup(p.stop)
def tearDown(self):
self.tmp.cleanup()
def batch(self):
s3 = FakeS3(entrance_objects(*UUIDS))
return main.collect_ready_sets(s3, self.cfg), s3
def fail_first_with(self, exc):
def process_one(cur, s3, cfg, s):
self.processed.append(s.uuid)
if s.uuid == UUIDS[0]:
raise exc
return mock.patch.object(main, "process_one", process_one)
def test_transient_s3_error_marks_image_error_and_aborts_batch(self):
sets, s3 = self.batch()
stats = main.PollerStats(None)
with self.fail_first_with(client_error("ServiceUnavailable", 503, "GetObject")), \
contextlib.redirect_stdout(io.StringIO()) as out:
with self.assertRaises(ClientError):
main._process_ready_batch(self.cfg, sets, s3, stats)
self.assertEqual(self.processed, [UUIDS[0]])
self.assertEqual([(u, st) for u, st, _ in self.status_calls], [(UUIDS[0], "ERROR")])
self.assertIn(f"[RESULT] uuid={UUIDS[0]} result=failure status=ERROR", out.getvalue())
self.assertEqual((stats.images_ok, stats.images_failed), (0, 1))
def test_bad_image_does_not_stop_batch(self):
sets, s3 = self.batch()
stats = main.PollerStats(None)
with self.fail_first_with(ValueError("metadata.servertime missing")), \
contextlib.redirect_stdout(io.StringIO()):
main._process_ready_batch(self.cfg, sets, s3, stats)
self.assertEqual(self.processed, UUIDS)
self.assertEqual(self.status_calls, [(UUIDS[0], "ERROR", "metadata.servertime missing")])
self.assertEqual((stats.images_ok, stats.images_failed), (2, 1))
def test_db_down_propagates_as_transient(self):
sets, s3 = self.batch()
def no_db(cfg):
raise psycopg.OperationalError("connection timeout expired")
with mock.patch.object(main, "pg_connect", no_db):
with self.assertRaises(psycopg.OperationalError) as ctx:
main._process_ready_batch(self.cfg, sets, s3)
self.assertTrue(main.is_transient_error(ctx.exception))
def test_run_once_parallel_raises_when_a_worker_hits_outage(self):
cfg = make_cfg(self.tmp.name, parallel_workers=3)
s3 = FakeS3(entrance_objects(*UUIDS))
with self.fail_first_with(s3_unavailable()), contextlib.redirect_stdout(io.StringIO()):
with self.assertRaises(ClientError):
main.run_once(cfg, main.PollerStats(None), s3c=s3)
self.assertEqual(sorted(self.processed), UUIDS) # die anderen Worker laufen zu Ende
class S3ClientConfigTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cfg = make_cfg(self.tmp.name)
def tearDown(self):
self.tmp.cleanup()
def test_poller_client_has_explicit_timeouts_and_retries(self):
conf = main.S3(self.cfg, main.s3_client_config(self.cfg)).client.meta.config
self.assertEqual(conf.connect_timeout, 10)
self.assertEqual(conf.read_timeout, 30)
self.assertEqual(conf.retries.get("mode"), "standard")
self.assertEqual(conf.retries.get("total_max_attempts"), 4)
self.assertNotIn("max_attempts", conf.retries)
self.assertEqual(conf.signature_version, "s3v4")
self.assertEqual(conf.s3.get("addressing_style"), "path")
def test_default_client_unchanged_for_gundi_path(self):
import boto3
from botocore.config import Config
# exakt wie vor der Aenderung gebaut
before = boto3.client(
"s3",
endpoint_url=self.cfg.s3_endpoint,
aws_access_key_id=self.cfg.s3_access_key,
aws_secret_access_key=self.cfg.s3_secret_key,
config=Config(max_pool_connections=self.cfg.s3_max_pool_connections),
).meta.config
now = main.S3(self.cfg).client.meta.config
for attr in ("connect_timeout", "read_timeout", "retries", "max_pool_connections",
"signature_version", "s3", "region_name"):
with self.subTest(attr=attr):
self.assertEqual(getattr(now, attr), getattr(before, attr))
self.assertEqual(now.retries, {"mode": "legacy"})
if __name__ == "__main__":
unittest.main()