Make legacy poller resilient to S3/DB outages

The poller crashed on every S3 or DB error (1162 restarts during the
fsn1 object storage degradation on 2026-09-21). Now:

- main loop catches errors per cycle, classifies them (S3 5xx/throttling,
  connection/timeouts, psycopg.OperationalError = transient; everything
  else logged with traceback) and retries with exponential backoff
  (5 s .. 5 min, reset after success); db_init runs until it succeeds once
- explicit botocore config for the poller: connect/read timeouts, standard
  retries (total attempts), s3v4, path addressing; S3(cfg) without a
  config keeps the previous client for the Gundi path
- DB connect timeout + TCP keepalives
- one S3 client per cycle shared by all workers
- a batch stops after a transient S3/DB error (image stays ERROR as before)
- no busy loop when ready sets keep failing
- [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck
- tools/entrance_inventory.py: read-only listing of icu/entrance orphans
- unit tests with simulated S3/DB failures

Per-image transaction, status names/order, keys and paths are unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-09-21 16:56:55 +02:00
co-authored by Claude Opus 5
parent 2fc8dea10c
commit 0f574cccde
6 changed files with 1023 additions and 18 deletions
+318 -18
View File
@@ -6,6 +6,8 @@ import json
import time
import tempfile
import subprocess
import threading
import traceback
import uuid as uuidlib
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
@@ -17,6 +19,12 @@ from urllib import request as urlrequest
import boto3
import psycopg
from botocore.config import Config as BotoClientConfig
from botocore.exceptions import (
ClientError,
ConnectionError as BotoConnectionError,
HTTPClientError,
IncompleteReadError,
)
from dateutil import tz
from PIL import Image, ImageOps, ImageEnhance
import pytesseract
@@ -73,6 +81,17 @@ class AppConfig:
deermapper_api_timeout_seconds: int = 20
deermapper_api_image_field: str = "image"
# Robustheit Poller: S3-Timeouts/Retries, DB-Connect-Timeout, Backoff, Lebenszeichen
s3_connect_timeout_seconds: float = 10.0
s3_read_timeout_seconds: float = 30.0
s3_retry_mode: str = "standard"
s3_max_attempts: int = 4
pg_connect_timeout_seconds: int = 10
backoff_initial_seconds: float = 5.0
backoff_max_seconds: float = 300.0
heartbeat_seconds: int = 300
heartbeat_file: str = "/tmp/melesicumover.heartbeat"
def load_config(path: str) -> AppConfig:
with open(path, "r", encoding="utf-8") as f:
@@ -128,6 +147,15 @@ def load_config(path: str) -> AppConfig:
deermapper_api_key=env_or("DEERMAPPER_API_KEY", deermapper.get("apiKey", "")),
deermapper_api_timeout_seconds=int(app.get("deermapper_api_timeout_seconds", 20)),
deermapper_api_image_field=app.get("deermapper_api_image_field", "image"),
s3_connect_timeout_seconds=float(app.get("s3_connect_timeout_seconds", 10)),
s3_read_timeout_seconds=float(app.get("s3_read_timeout_seconds", 30)),
s3_retry_mode=str(app.get("s3_retry_mode", "standard")),
s3_max_attempts=max(1, int(app.get("s3_max_attempts", 4))),
pg_connect_timeout_seconds=max(1, int(app.get("pg_connect_timeout_seconds", 10))),
backoff_initial_seconds=max(0.1, float(app.get("backoff_initial_seconds", 5))),
backoff_max_seconds=max(1.0, float(app.get("backoff_max_seconds", 300))),
heartbeat_seconds=max(10, int(app.get("heartbeat_seconds", 300))),
heartbeat_file=app.get("heartbeat_file", "/tmp/melesicumover.heartbeat"),
)
# Fail-fast: Secrets kommen ausschliesslich aus der Umgebung (.env). Fehlen sie,
@@ -454,6 +482,26 @@ def db_init(cur):
""")
def pg_connect(cfg: AppConfig) -> psycopg.Connection:
# connect_timeout + TCP-Keepalives: ein weggebrochener DB-Host blockiert den
# Poller nicht unbegrenzt, sondern endet als OperationalError (-> Backoff).
return psycopg.connect(
cfg.pg_dsn,
connect_timeout=cfg.pg_connect_timeout_seconds,
keepalives=1,
keepalives_idle=60,
keepalives_interval=10,
keepalives_count=3,
)
def init_db_schema(cfg: AppConfig):
with pg_connect(cfg) as db:
with db.cursor() as cur:
db_init(cur)
db.commit()
def job_upsert_new(cur, uuid: str, jpg_key: str, meta_key: str, cat_key: Optional[str]):
cur.execute("""
INSERT INTO remote_cam.import_job (image_uuid, status, entrance_jpg_key, entrance_meta_key, entrance_cat_key, has_categories)
@@ -676,15 +724,34 @@ def insert_api_response(cur, uuid: str, success: bool, message: str):
# -----------------------
# S3 ops wrapper
# -----------------------
def s3_client_config(cfg: AppConfig) -> BotoClientConfig:
"""Explizite Timeouts/Retries fuer den Poller (botocore-Default: legacy, 60 s Read-Timeout).
Pfad-Adressierung + s3v4 wie im Portal; die Adressierung entspricht dem bisherigen
Verhalten mit eigenem endpoint_url.
"""
return BotoClientConfig(
max_pool_connections=cfg.s3_max_pool_connections,
connect_timeout=cfg.s3_connect_timeout_seconds,
read_timeout=cfg.s3_read_timeout_seconds,
# total_max_attempts = Versuche inkl. erstem (botocores "max_attempts" zaehlt nur Wiederholungen)
retries={"mode": cfg.s3_retry_mode, "total_max_attempts": cfg.s3_max_attempts},
signature_version="s3v4",
s3={"addressing_style": "path"},
)
class S3:
def __init__(self, cfg: AppConfig):
def __init__(self, cfg: AppConfig, client_config: Optional[BotoClientConfig] = None):
self.cfg = cfg
# Ohne client_config bleibt die bisherige Client-Konfiguration (so nutzt sie der
# Gundi-Pfad); der Poller uebergibt s3_client_config(cfg).
self.client = boto3.client(
"s3",
endpoint_url=cfg.s3_endpoint,
aws_access_key_id=cfg.s3_access_key,
aws_secret_access_key=cfg.s3_secret_key,
config=BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
config=client_config or BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
)
def list_entrance_objects(self):
@@ -933,15 +1000,15 @@ def cleanup_entrance_set(s3: S3, s: EntranceSet) -> int:
return s3.delete_keys(to_delete)
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
s3c = S3(cfg)
with psycopg.connect(cfg.pg_dsn) as db:
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet], s3c: S3, stats: Optional["PollerStats"] = None):
with pg_connect(cfg) as db:
for s in batch:
start = time.perf_counter()
result = "failure"
final_status = "UNKNOWN"
err_text = None
cleanup_deleted = None
abort = None
try:
skip_duplicate = False
with db.transaction():
@@ -963,6 +1030,11 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
job_set_status(cur, s.uuid, "ERROR", err=str(e))
final_status = "ERROR"
err_text = str(e)
# S3/DB gestoert: Bild bleibt wie bisher auf ERROR, der Rest des Batches wird
# nicht mehr angefasst (naechster Zyklus nach Backoff) statt jedes Bild
# einzeln gegen die Stoerung laufen zu lassen.
if is_transient_error(e):
abort = e
elapsed_ms = int((time.perf_counter() - start) * 1000)
if err_text:
print(
@@ -976,19 +1048,25 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
)
else:
print(f"[RESULT] uuid={s.uuid} result={result} status={final_status} process_ms={elapsed_ms}")
if stats is not None:
stats.image_done(result == "success")
if abort is not None:
raise abort
if cfg.job_sleep_seconds > 0:
time.sleep(cfg.job_sleep_seconds)
def run_once(cfg: AppConfig) -> int:
s3c = S3(cfg)
def run_once(cfg: AppConfig, stats: Optional["PollerStats"] = None, s3c: Optional[S3] = None) -> int:
# Ein Client pro Zyklus fuer alle Worker: boto3-Clients sind threadsicher, das
# parallele Anlegen ueber die Default-Session dagegen nicht.
s3c = s3c or S3(cfg, s3_client_config(cfg))
ready = collect_ready_sets(s3c, cfg)
if not ready:
return 0
workers = max(1, min(cfg.parallel_workers, len(ready)))
if workers == 1:
_process_ready_batch(cfg, ready)
_process_ready_batch(cfg, ready, s3c, stats)
return len(ready)
batches: list[list[EntranceSet]] = [[] for _ in range(workers)]
@@ -996,13 +1074,237 @@ def run_once(cfg: AppConfig) -> int:
batches[i % workers].append(s)
with ThreadPoolExecutor(max_workers=workers) as pool:
futures = [pool.submit(_process_ready_batch, cfg, batch) for batch in batches if batch]
futures = [pool.submit(_process_ready_batch, cfg, batch, s3c, stats) for batch in batches if batch]
for future in as_completed(futures):
future.result()
return len(ready)
# -----------------------
# Robustheit: Fehlerklassen, Backoff, Lebenszeichen
# -----------------------
TRANSIENT_S3_CODES = {
"ServiceUnavailable", "SlowDown", "Throttling", "ThrottlingException", "TooManyRequests",
"RequestLimitExceeded", "RequestTimeout", "InternalError", "500", "502", "503", "504",
}
def is_transient_error(exc: Optional[BaseException]) -> bool:
"""True fuer Stoerungen, die ohne Codeaenderung vorbeigehen: S3 5xx/Throttling,
Verbindungs-/Timeout-Fehler, DB weg (psycopg.OperationalError). Alles andere
(4xx wie AccessDenied, Programmierfehler, SQL-Fehler) gilt als unerwartet.
Folgt der expliziten Ursache (raise ... from ...)."""
seen = set()
while exc is not None and id(exc) not in seen:
seen.add(id(exc))
if isinstance(exc, ClientError):
err = exc.response.get("Error") or {}
http = (exc.response.get("ResponseMetadata") or {}).get("HTTPStatusCode") or 0
if str(err.get("Code", "")) in TRANSIENT_S3_CODES or http >= 500 or http == 429:
return True
elif isinstance(exc, (BotoConnectionError, HTTPClientError, IncompleteReadError)):
return True
elif isinstance(exc, psycopg.OperationalError):
return True
elif isinstance(exc, (ConnectionError, TimeoutError)):
return True
exc = exc.__cause__
return False
def describe_error(exc: BaseException, limit: int = 300) -> str:
text = " ".join(str(exc).split())
if len(text) > limit:
text = text[:limit] + "..."
return f"{type(exc).__name__}: {text}"
class Backoff:
"""Exponentiell: initial, 2x, 4x ... bis maximum; reset() nach einem erfolgreichen Zyklus."""
def __init__(self, initial: float, maximum: float, factor: float = 2.0):
self.initial = initial
self.maximum = maximum
self.factor = factor
self.failures = 0
def next_delay(self) -> float:
# Exponent begrenzen: bei tagelanger Stoerung sonst OverflowError.
delay = min(self.maximum, self.initial * (self.factor ** min(self.failures, 32)))
self.failures += 1
return delay
def reset(self):
self.failures = 0
def _utc_iso(ts: Optional[float]) -> str:
if ts is None:
return "never"
return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
class PollerStats:
"""Zaehler fuer Lebenszeichen-Log und Healthcheck. Threadsicher, weil die Worker je Bild melden.
Die Heartbeat-Datei wird bei jedem Zyklus und jedem Bild neu geschrieben; ihr mtime
zeigt dem Docker-Healthcheck, dass die Schleife vorankommt (wartet vs. haengt)."""
def __init__(self, heartbeat_file: Optional[str], clock=time.time):
self._lock = threading.Lock()
self._clock = clock
self.heartbeat_file = heartbeat_file
self.started = clock()
self.cycles = 0
self.cycles_failed = 0
self.images_ok = 0
self.images_failed = 0
self.errors_transient = 0
self.errors_other = 0
self.consecutive_failures = 0
self.last_success: Optional[float] = None
self.last_error: Optional[str] = None
def image_done(self, ok: bool):
with self._lock:
if ok:
self.images_ok += 1
else:
self.images_failed += 1
self._write_heartbeat()
def cycle_ok(self):
with self._lock:
self.cycles += 1
self.consecutive_failures = 0
self.last_success = self._clock()
self._write_heartbeat()
def cycle_failed(self, transient: bool, message: str):
with self._lock:
self.cycles += 1
self.cycles_failed += 1
self.consecutive_failures += 1
if transient:
self.errors_transient += 1
else:
self.errors_other += 1
self.last_error = message
self._write_heartbeat()
def touch(self):
with self._lock:
self._write_heartbeat()
def snapshot(self) -> dict:
now = self._clock()
return {
"ts": _utc_iso(now),
"uptime_s": int(now - self.started),
"cycles": self.cycles,
"cycles_failed": self.cycles_failed,
"consecutive_failures": self.consecutive_failures,
"images_ok": self.images_ok,
"images_failed": self.images_failed,
"errors_transient": self.errors_transient,
"errors_other": self.errors_other,
"last_success": _utc_iso(self.last_success),
"last_success_age_s": None if self.last_success is None else int(now - self.last_success),
"last_error": self.last_error,
}
def summary(self) -> str:
s = self.snapshot()
age = "-" if s["last_success_age_s"] is None else f"{s['last_success_age_s']}s"
return (
f"uptime={s['uptime_s']}s cycles={s['cycles']} cycles_failed={s['cycles_failed']} "
f"consecutive_failures={s['consecutive_failures']} images_ok={s['images_ok']} "
f"images_failed={s['images_failed']} errors_transient={s['errors_transient']} "
f"errors_other={s['errors_other']} last_success={s['last_success']} last_success_age={age}"
)
def _write_heartbeat(self):
if not self.heartbeat_file:
return
try:
tmp = f"{self.heartbeat_file}.tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(self.snapshot(), f)
os.replace(tmp, self.heartbeat_file)
except OSError as e:
print(f"[WARN] heartbeat file not writable: {e}")
def poll_forever(
cfg: AppConfig,
*,
cycle=run_once,
init_schema=init_db_schema,
sleep=time.sleep,
clock=time.time,
stats: Optional[PollerStats] = None,
max_iterations: Optional[int] = None,
) -> PollerStats:
"""Hauptschleife. Faengt Fehler je Zyklus ab und wartet mit Backoff - S3- oder DB-Ausfaelle
beenden den Prozess nicht mehr. db_init laeuft einmal erfolgreich, danach nicht erneut.
Transaktion pro Bild und Statusfolge liegen unveraendert in _process_ready_batch/process_one.
"""
stats = stats or PollerStats(cfg.heartbeat_file, clock=clock)
backoff = Backoff(cfg.backoff_initial_seconds, cfg.backoff_max_seconds)
schema_ready = False
failing_since: Optional[float] = None
last_heartbeat_log = clock()
iterations = 0
while max_iterations is None or iterations < max_iterations:
iterations += 1
stats.touch()
ok_before = stats.images_ok
try:
if not schema_ready:
init_schema(cfg)
schema_ready = True
n = cycle(cfg, stats)
except Exception as e:
transient = is_transient_error(e)
msg = describe_error(e)
delay = backoff.next_delay()
stats.cycle_failed(transient, msg)
if failing_since is None:
failing_since = clock()
kind = "transient" if transient else "unexpected"
print(
f"[CYCLE_FAIL] kind={kind} consecutive={backoff.failures} "
f"retry_in={delay:.0f}s error={msg}"
)
if not transient:
traceback.print_exc()
n = None
else:
if failing_since is not None:
print(
f"[RECOVERED] after_failures={backoff.failures} "
f"outage_s={int(clock() - failing_since)}"
)
failing_since = None
backoff.reset()
stats.cycle_ok()
delay = None
if clock() - last_heartbeat_log >= cfg.heartbeat_seconds:
print(f"[HEARTBEAT] {stats.summary()}")
last_heartbeat_log = clock()
if delay is not None:
sleep(delay)
elif n == 0 or stats.images_ok == ok_before:
# Nichts zu tun - oder nur Saetze, die erneut scheitern: nicht im Leerlauf kreiseln.
sleep(cfg.poll_seconds)
return stats
def main():
if os.environ.get("DEBUG") == "1":
import debugpy
@@ -1013,15 +1315,13 @@ def main():
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
cfg = load_config(cfg_path)
with psycopg.connect(cfg.pg_dsn) as db:
with db.cursor() as cur:
db_init(cur)
db.commit()
while True:
n = run_once(cfg)
if n == 0:
time.sleep(cfg.poll_seconds)
print(
f"[START] poll={cfg.poll_seconds}s workers={cfg.parallel_workers} "
f"s3_timeout=connect:{cfg.s3_connect_timeout_seconds:g}s/read:{cfg.s3_read_timeout_seconds:g}s "
f"s3_retries={cfg.s3_retry_mode}/{cfg.s3_max_attempts} pg_connect_timeout={cfg.pg_connect_timeout_seconds}s "
f"backoff={cfg.backoff_initial_seconds:g}-{cfg.backoff_max_seconds:g}s heartbeat={cfg.heartbeat_seconds}s"
)
poll_forever(cfg)
if __name__ == "__main__":