The poller crashed on every S3 or DB error (1162 restarts during the fsn1 object storage degradation on 2026-09-21). Now: - main loop catches errors per cycle, classifies them (S3 5xx/throttling, connection/timeouts, psycopg.OperationalError = transient; everything else logged with traceback) and retries with exponential backoff (5 s .. 5 min, reset after success); db_init runs until it succeeds once - explicit botocore config for the poller: connect/read timeouts, standard retries (total attempts), s3v4, path addressing; S3(cfg) without a config keeps the previous client for the Gundi path - DB connect timeout + TCP keepalives - one S3 client per cycle shared by all workers - a batch stops after a transient S3/DB error (image stays ERROR as before) - no busy loop when ready sets keep failing - [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck - tools/entrance_inventory.py: read-only listing of icu/entrance orphans - unit tests with simulated S3/DB failures Per-image transaction, status names/order, keys and paths are unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
441 lines
16 KiB
Python
441 lines
16 KiB
Python
"""Backoff und Fehlerbehandlung des Legacy-Pollers mit simulierten S3-/DB-Stoerungen.
|
|
|
|
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
|
|
|
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
|
"""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import psycopg # noqa: E402
|
|
from botocore.exceptions import ( # noqa: E402
|
|
ClientError,
|
|
ConnectTimeoutError,
|
|
ConnectionClosedError,
|
|
EndpointConnectionError,
|
|
ReadTimeoutError,
|
|
)
|
|
|
|
import main # noqa: E402
|
|
|
|
|
|
def client_error(code: str, http: int, op: str = "ListObjectsV2") -> ClientError:
|
|
return ClientError({"Error": {"Code": code, "Message": code}, "ResponseMetadata": {"HTTPStatusCode": http}}, op)
|
|
|
|
|
|
def s3_unavailable() -> ClientError:
|
|
return client_error("ServiceUnavailable", 503)
|
|
|
|
|
|
def make_cfg(tmpdir: str, **overrides) -> main.AppConfig:
|
|
params = dict(
|
|
s3_endpoint="https://s3.invalid/",
|
|
s3_access_key="test",
|
|
s3_secret_key="test",
|
|
s3_bucket="bucket",
|
|
pg_dsn="postgresql://invalid/db",
|
|
poll_seconds=30,
|
|
parallel_workers=1,
|
|
backoff_initial_seconds=5,
|
|
backoff_max_seconds=300,
|
|
heartbeat_seconds=300,
|
|
heartbeat_file=os.path.join(tmpdir, "heartbeat.json"),
|
|
)
|
|
params.update(overrides)
|
|
return main.AppConfig(**params)
|
|
|
|
|
|
class FakeClock:
|
|
"""Zeit laeuft nur durch sleep() - Tests laufen ohne echtes Warten."""
|
|
|
|
def __init__(self, start: float = 1_000_000.0):
|
|
self.now = start
|
|
self.sleeps: list[float] = []
|
|
|
|
def time(self) -> float:
|
|
return self.now
|
|
|
|
def sleep(self, seconds: float):
|
|
self.sleeps.append(seconds)
|
|
self.now += seconds
|
|
|
|
|
|
class ScriptedCycle:
|
|
"""Zyklus-Ersatz: arbeitet eine Liste ab (Exception -> werfen, int -> zurueckgeben)."""
|
|
|
|
def __init__(self, steps):
|
|
self.steps = list(steps)
|
|
self.calls = 0
|
|
|
|
def __call__(self, cfg, stats):
|
|
self.calls += 1
|
|
step = self.steps.pop(0) if self.steps else 0
|
|
if isinstance(step, BaseException):
|
|
raise step
|
|
if callable(step):
|
|
return step(stats)
|
|
return step
|
|
|
|
|
|
class FakeS3:
|
|
"""Simulierter S3-Client: Listing liefert Objekte oder wirft den angegebenen Fehler."""
|
|
|
|
def __init__(self, objects=None, list_error: BaseException = None):
|
|
self.objects = objects or []
|
|
self.list_error = list_error
|
|
|
|
def list_entrance_objects(self):
|
|
if self.list_error is not None:
|
|
raise self.list_error
|
|
yield from self.objects
|
|
|
|
|
|
class FakeConn:
|
|
"""Simulierte psycopg-Verbindung; die SQL-Funktionen werden in den Tests gepatcht."""
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
@contextlib.contextmanager
|
|
def transaction(self):
|
|
yield
|
|
|
|
def cursor(self):
|
|
return contextlib.nullcontext(object())
|
|
|
|
|
|
def entrance_objects(*uuids):
|
|
old = datetime.now(timezone.utc) - timedelta(hours=1)
|
|
objs = []
|
|
for u in uuids:
|
|
objs.append({"Key": f"icu/entrance/{u}.jpg", "LastModified": old})
|
|
objs.append({"Key": f"icu/entrance/{u}.json", "LastModified": old})
|
|
return objs
|
|
|
|
|
|
UUIDS = [f"00000000-0000-4000-8000-00000000000{i}" for i in range(1, 4)]
|
|
|
|
|
|
class BackoffTest(unittest.TestCase):
|
|
def test_doubles_up_to_maximum(self):
|
|
b = main.Backoff(5, 300)
|
|
self.assertEqual([b.next_delay() for _ in range(9)], [5, 10, 20, 40, 80, 160, 300, 300, 300])
|
|
|
|
def test_reset_starts_again_at_initial(self):
|
|
b = main.Backoff(5, 300)
|
|
for _ in range(4):
|
|
b.next_delay()
|
|
b.reset()
|
|
self.assertEqual(b.failures, 0)
|
|
self.assertEqual(b.next_delay(), 5)
|
|
|
|
def test_long_outage_does_not_overflow(self):
|
|
b = main.Backoff(5, 300)
|
|
for _ in range(5000):
|
|
delay = b.next_delay()
|
|
self.assertEqual(delay, 300)
|
|
|
|
|
|
class ErrorClassificationTest(unittest.TestCase):
|
|
def test_transient(self):
|
|
cases = [
|
|
s3_unavailable(),
|
|
client_error("SlowDown", 503),
|
|
client_error("InternalError", 500),
|
|
client_error("TooManyRequests", 429),
|
|
client_error("Whatever", 502),
|
|
EndpointConnectionError(endpoint_url="https://s3.invalid"),
|
|
ConnectTimeoutError(endpoint_url="https://s3.invalid"),
|
|
ReadTimeoutError(endpoint_url="https://s3.invalid"),
|
|
ConnectionClosedError(endpoint_url="https://s3.invalid"),
|
|
psycopg.OperationalError("connection refused"),
|
|
psycopg.errors.AdminShutdown("terminating connection due to administrator command"),
|
|
ConnectionResetError(),
|
|
TimeoutError(),
|
|
]
|
|
for exc in cases:
|
|
with self.subTest(exc=repr(exc)):
|
|
self.assertTrue(main.is_transient_error(exc))
|
|
|
|
def test_unexpected(self):
|
|
cases = [
|
|
client_error("AccessDenied", 403),
|
|
client_error("NoSuchKey", 404, "GetObject"),
|
|
client_error("NoSuchBucket", 404),
|
|
psycopg.ProgrammingError("syntax error"),
|
|
psycopg.errors.InsufficientPrivilege("permission denied"),
|
|
KeyError("status"),
|
|
ValueError("metadata.servertime missing"),
|
|
]
|
|
for exc in cases:
|
|
with self.subTest(exc=repr(exc)):
|
|
self.assertFalse(main.is_transient_error(exc))
|
|
|
|
def test_follows_explicit_cause_only(self):
|
|
try:
|
|
try:
|
|
raise s3_unavailable()
|
|
except ClientError as e:
|
|
raise RuntimeError("wrapped") from e
|
|
except RuntimeError as wrapped:
|
|
self.assertTrue(main.is_transient_error(wrapped))
|
|
|
|
# Ein Bug im Fehlerpfad (nur __context__) bleibt ein unerwarteter Fehler.
|
|
try:
|
|
try:
|
|
raise s3_unavailable()
|
|
except ClientError:
|
|
raise KeyError("bug in handler")
|
|
except KeyError as bug:
|
|
self.assertFalse(main.is_transient_error(bug))
|
|
|
|
|
|
class PollForeverTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.cfg = make_cfg(self.tmp.name)
|
|
self.clock = FakeClock()
|
|
self.out = io.StringIO()
|
|
self.err = io.StringIO()
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def run_loop(self, cycle, iterations, init_schema=lambda cfg: None):
|
|
with contextlib.redirect_stdout(self.out), contextlib.redirect_stderr(self.err):
|
|
return main.poll_forever(
|
|
self.cfg,
|
|
cycle=cycle,
|
|
init_schema=init_schema,
|
|
sleep=self.clock.sleep,
|
|
clock=self.clock.time,
|
|
max_iterations=iterations,
|
|
)
|
|
|
|
def test_s3_outage_backs_off_and_recovers(self):
|
|
cycle = ScriptedCycle([s3_unavailable(), s3_unavailable(), s3_unavailable(), 0, 0, s3_unavailable(), 0])
|
|
stats = self.run_loop(cycle, 7)
|
|
self.assertEqual(self.clock.sleeps, [5, 10, 20, 30, 30, 5, 30])
|
|
self.assertEqual(stats.cycles, 7)
|
|
self.assertEqual(stats.cycles_failed, 4)
|
|
self.assertEqual(stats.errors_transient, 4)
|
|
self.assertEqual(stats.errors_other, 0)
|
|
self.assertEqual(stats.consecutive_failures, 0)
|
|
self.assertIsNotNone(stats.last_success)
|
|
log = self.out.getvalue()
|
|
self.assertIn("[CYCLE_FAIL] kind=transient consecutive=1 retry_in=5s error=ClientError", log)
|
|
self.assertIn("[RECOVERED] after_failures=3 outage_s=35", log)
|
|
self.assertNotIn("Traceback", self.err.getvalue())
|
|
|
|
def test_backoff_is_capped_during_long_outage(self):
|
|
stats = self.run_loop(ScriptedCycle([s3_unavailable()] * 12), 12)
|
|
self.assertEqual(self.clock.sleeps[-4:], [300, 300, 300, 300])
|
|
self.assertEqual(stats.consecutive_failures, 12)
|
|
|
|
def test_db_unavailable_at_start_is_retried_then_schema_init_runs_once(self):
|
|
calls = []
|
|
|
|
def init_schema(cfg):
|
|
calls.append(1)
|
|
if len(calls) <= 2:
|
|
raise psycopg.OperationalError("could not connect to server")
|
|
|
|
cycle = ScriptedCycle([0, 0, 0])
|
|
stats = self.run_loop(cycle, 5, init_schema=init_schema)
|
|
self.assertEqual(len(calls), 3)
|
|
self.assertEqual(cycle.calls, 3)
|
|
self.assertEqual(self.clock.sleeps, [5, 10, 30, 30, 30])
|
|
self.assertEqual(stats.errors_transient, 2)
|
|
|
|
def test_unexpected_error_is_logged_with_traceback_but_does_not_crash(self):
|
|
cycle = ScriptedCycle([KeyError("boom"), KeyError("boom"), 0])
|
|
stats = self.run_loop(cycle, 3)
|
|
self.assertEqual(stats.errors_other, 2)
|
|
self.assertEqual(self.clock.sleeps, [5, 10, 30])
|
|
self.assertIn("[CYCLE_FAIL] kind=unexpected", self.out.getvalue())
|
|
self.assertIn("Traceback", self.err.getvalue())
|
|
self.assertIn("KeyError: 'boom'", self.err.getvalue())
|
|
|
|
def test_no_busy_loop_when_ready_sets_keep_failing(self):
|
|
def progress(stats):
|
|
stats.image_done(True)
|
|
return 1
|
|
|
|
cycle = ScriptedCycle([2, progress, 0])
|
|
self.run_loop(cycle, 3)
|
|
# 2 Saetze ohne Erfolg -> Poll-Pause; Fortschritt -> sofort weiter; leer -> Poll-Pause
|
|
self.assertEqual(self.clock.sleeps, [30, 30])
|
|
|
|
def test_heartbeat_log_and_file(self):
|
|
stats = self.run_loop(ScriptedCycle([0] * 12), 12)
|
|
beats = [line for line in self.out.getvalue().splitlines() if line.startswith("[HEARTBEAT]")]
|
|
self.assertEqual(len(beats), 1) # Zyklus 11 laeuft bei +300 s -> genau eine Zeile
|
|
self.assertIn("cycles=11 cycles_failed=0", beats[0])
|
|
self.assertIn("last_success_age=0s", beats[0])
|
|
with open(self.cfg.heartbeat_file, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
self.assertEqual(data["cycles"], stats.cycles)
|
|
self.assertEqual(data["errors_transient"], 0)
|
|
|
|
|
|
class RunOnceWithFakeS3Test(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.cfg = make_cfg(self.tmp.name)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def test_listing_failure_propagates_to_loop_which_survives(self):
|
|
s3 = FakeS3(list_error=s3_unavailable())
|
|
with self.assertRaises(ClientError):
|
|
main.run_once(self.cfg, None, s3c=s3)
|
|
|
|
clock = FakeClock()
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
stats = main.poll_forever(
|
|
self.cfg,
|
|
cycle=lambda cfg, st: main.run_once(cfg, st, s3c=s3),
|
|
init_schema=lambda cfg: None,
|
|
sleep=clock.sleep,
|
|
clock=clock.time,
|
|
max_iterations=3,
|
|
)
|
|
self.assertEqual(clock.sleeps, [5, 10, 20])
|
|
self.assertEqual(stats.errors_transient, 3)
|
|
|
|
def test_empty_entrance_returns_zero(self):
|
|
self.assertEqual(main.run_once(self.cfg, None, s3c=FakeS3()), 0)
|
|
|
|
|
|
class ProcessBatchTest(unittest.TestCase):
|
|
"""Transaktion pro Bild bleibt; bei S3-/DB-Stoerung wird der Rest des Batches nicht angefasst."""
|
|
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.cfg = make_cfg(self.tmp.name)
|
|
self.status_calls = []
|
|
self.processed = []
|
|
patches = [
|
|
mock.patch.object(main, "pg_connect", lambda cfg: FakeConn()),
|
|
mock.patch.object(main, "is_completed_job_materialized", lambda cur, s3, cfg, uuid: False),
|
|
mock.patch.object(main, "job_upsert_new", lambda cur, uuid, *a: None),
|
|
mock.patch.object(main, "job_get", lambda cur, uuid: {"status": "CLEANED"}),
|
|
mock.patch.object(
|
|
main, "job_set_status",
|
|
lambda cur, uuid, status, **kw: self.status_calls.append((uuid, status, kw.get("err"))),
|
|
),
|
|
]
|
|
for p in patches:
|
|
p.start()
|
|
self.addCleanup(p.stop)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def batch(self):
|
|
s3 = FakeS3(entrance_objects(*UUIDS))
|
|
return main.collect_ready_sets(s3, self.cfg), s3
|
|
|
|
def fail_first_with(self, exc):
|
|
def process_one(cur, s3, cfg, s):
|
|
self.processed.append(s.uuid)
|
|
if s.uuid == UUIDS[0]:
|
|
raise exc
|
|
return mock.patch.object(main, "process_one", process_one)
|
|
|
|
def test_transient_s3_error_marks_image_error_and_aborts_batch(self):
|
|
sets, s3 = self.batch()
|
|
stats = main.PollerStats(None)
|
|
with self.fail_first_with(client_error("ServiceUnavailable", 503, "GetObject")), \
|
|
contextlib.redirect_stdout(io.StringIO()) as out:
|
|
with self.assertRaises(ClientError):
|
|
main._process_ready_batch(self.cfg, sets, s3, stats)
|
|
self.assertEqual(self.processed, [UUIDS[0]])
|
|
self.assertEqual([(u, st) for u, st, _ in self.status_calls], [(UUIDS[0], "ERROR")])
|
|
self.assertIn(f"[RESULT] uuid={UUIDS[0]} result=failure status=ERROR", out.getvalue())
|
|
self.assertEqual((stats.images_ok, stats.images_failed), (0, 1))
|
|
|
|
def test_bad_image_does_not_stop_batch(self):
|
|
sets, s3 = self.batch()
|
|
stats = main.PollerStats(None)
|
|
with self.fail_first_with(ValueError("metadata.servertime missing")), \
|
|
contextlib.redirect_stdout(io.StringIO()):
|
|
main._process_ready_batch(self.cfg, sets, s3, stats)
|
|
self.assertEqual(self.processed, UUIDS)
|
|
self.assertEqual(self.status_calls, [(UUIDS[0], "ERROR", "metadata.servertime missing")])
|
|
self.assertEqual((stats.images_ok, stats.images_failed), (2, 1))
|
|
|
|
def test_db_down_propagates_as_transient(self):
|
|
sets, s3 = self.batch()
|
|
|
|
def no_db(cfg):
|
|
raise psycopg.OperationalError("connection timeout expired")
|
|
|
|
with mock.patch.object(main, "pg_connect", no_db):
|
|
with self.assertRaises(psycopg.OperationalError) as ctx:
|
|
main._process_ready_batch(self.cfg, sets, s3)
|
|
self.assertTrue(main.is_transient_error(ctx.exception))
|
|
|
|
def test_run_once_parallel_raises_when_a_worker_hits_outage(self):
|
|
cfg = make_cfg(self.tmp.name, parallel_workers=3)
|
|
s3 = FakeS3(entrance_objects(*UUIDS))
|
|
with self.fail_first_with(s3_unavailable()), contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaises(ClientError):
|
|
main.run_once(cfg, main.PollerStats(None), s3c=s3)
|
|
self.assertEqual(sorted(self.processed), UUIDS) # die anderen Worker laufen zu Ende
|
|
|
|
|
|
class S3ClientConfigTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.cfg = make_cfg(self.tmp.name)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def test_poller_client_has_explicit_timeouts_and_retries(self):
|
|
conf = main.S3(self.cfg, main.s3_client_config(self.cfg)).client.meta.config
|
|
self.assertEqual(conf.connect_timeout, 10)
|
|
self.assertEqual(conf.read_timeout, 30)
|
|
self.assertEqual(conf.retries.get("mode"), "standard")
|
|
self.assertEqual(conf.retries.get("total_max_attempts"), 4)
|
|
self.assertNotIn("max_attempts", conf.retries)
|
|
self.assertEqual(conf.signature_version, "s3v4")
|
|
self.assertEqual(conf.s3.get("addressing_style"), "path")
|
|
|
|
def test_default_client_unchanged_for_gundi_path(self):
|
|
import boto3
|
|
from botocore.config import Config
|
|
|
|
# exakt wie vor der Aenderung gebaut
|
|
before = boto3.client(
|
|
"s3",
|
|
endpoint_url=self.cfg.s3_endpoint,
|
|
aws_access_key_id=self.cfg.s3_access_key,
|
|
aws_secret_access_key=self.cfg.s3_secret_key,
|
|
config=Config(max_pool_connections=self.cfg.s3_max_pool_connections),
|
|
).meta.config
|
|
now = main.S3(self.cfg).client.meta.config
|
|
for attr in ("connect_timeout", "read_timeout", "retries", "max_pool_connections",
|
|
"signature_version", "s3", "region_name"):
|
|
with self.subTest(attr=attr):
|
|
self.assertEqual(getattr(now, attr), getattr(before, attr))
|
|
self.assertEqual(now.retries, {"mode": "legacy"})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|