Make legacy poller resilient to S3/DB outages
The poller crashed on every S3 or DB error (1162 restarts during the fsn1 object storage degradation on 2026-09-21). Now: - main loop catches errors per cycle, classifies them (S3 5xx/throttling, connection/timeouts, psycopg.OperationalError = transient; everything else logged with traceback) and retries with exponential backoff (5 s .. 5 min, reset after success); db_init runs until it succeeds once - explicit botocore config for the poller: connect/read timeouts, standard retries (total attempts), s3v4, path addressing; S3(cfg) without a config keeps the previous client for the Gundi path - DB connect timeout + TCP keepalives - one S3 client per cycle shared by all workers - a batch stops after a transient S3/DB error (image stays ERROR as before) - no busy loop when ready sets keep failing - [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck - tools/entrance_inventory.py: read-only listing of icu/entrance orphans - unit tests with simulated S3/DB failures Per-image transaction, status names/order, keys and paths are unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,8 @@ import json
|
||||
import time
|
||||
import tempfile
|
||||
import subprocess
|
||||
import threading
|
||||
import traceback
|
||||
import uuid as uuidlib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
@@ -17,6 +19,12 @@ from urllib import request as urlrequest
|
||||
import boto3
|
||||
import psycopg
|
||||
from botocore.config import Config as BotoClientConfig
|
||||
from botocore.exceptions import (
|
||||
ClientError,
|
||||
ConnectionError as BotoConnectionError,
|
||||
HTTPClientError,
|
||||
IncompleteReadError,
|
||||
)
|
||||
from dateutil import tz
|
||||
from PIL import Image, ImageOps, ImageEnhance
|
||||
import pytesseract
|
||||
@@ -73,6 +81,17 @@ class AppConfig:
|
||||
deermapper_api_timeout_seconds: int = 20
|
||||
deermapper_api_image_field: str = "image"
|
||||
|
||||
# Robustheit Poller: S3-Timeouts/Retries, DB-Connect-Timeout, Backoff, Lebenszeichen
|
||||
s3_connect_timeout_seconds: float = 10.0
|
||||
s3_read_timeout_seconds: float = 30.0
|
||||
s3_retry_mode: str = "standard"
|
||||
s3_max_attempts: int = 4
|
||||
pg_connect_timeout_seconds: int = 10
|
||||
backoff_initial_seconds: float = 5.0
|
||||
backoff_max_seconds: float = 300.0
|
||||
heartbeat_seconds: int = 300
|
||||
heartbeat_file: str = "/tmp/melesicumover.heartbeat"
|
||||
|
||||
|
||||
def load_config(path: str) -> AppConfig:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
@@ -128,6 +147,15 @@ def load_config(path: str) -> AppConfig:
|
||||
deermapper_api_key=env_or("DEERMAPPER_API_KEY", deermapper.get("apiKey", "")),
|
||||
deermapper_api_timeout_seconds=int(app.get("deermapper_api_timeout_seconds", 20)),
|
||||
deermapper_api_image_field=app.get("deermapper_api_image_field", "image"),
|
||||
s3_connect_timeout_seconds=float(app.get("s3_connect_timeout_seconds", 10)),
|
||||
s3_read_timeout_seconds=float(app.get("s3_read_timeout_seconds", 30)),
|
||||
s3_retry_mode=str(app.get("s3_retry_mode", "standard")),
|
||||
s3_max_attempts=max(1, int(app.get("s3_max_attempts", 4))),
|
||||
pg_connect_timeout_seconds=max(1, int(app.get("pg_connect_timeout_seconds", 10))),
|
||||
backoff_initial_seconds=max(0.1, float(app.get("backoff_initial_seconds", 5))),
|
||||
backoff_max_seconds=max(1.0, float(app.get("backoff_max_seconds", 300))),
|
||||
heartbeat_seconds=max(10, int(app.get("heartbeat_seconds", 300))),
|
||||
heartbeat_file=app.get("heartbeat_file", "/tmp/melesicumover.heartbeat"),
|
||||
)
|
||||
|
||||
# Fail-fast: Secrets kommen ausschliesslich aus der Umgebung (.env). Fehlen sie,
|
||||
@@ -454,6 +482,26 @@ def db_init(cur):
|
||||
""")
|
||||
|
||||
|
||||
def pg_connect(cfg: AppConfig) -> psycopg.Connection:
|
||||
# connect_timeout + TCP-Keepalives: ein weggebrochener DB-Host blockiert den
|
||||
# Poller nicht unbegrenzt, sondern endet als OperationalError (-> Backoff).
|
||||
return psycopg.connect(
|
||||
cfg.pg_dsn,
|
||||
connect_timeout=cfg.pg_connect_timeout_seconds,
|
||||
keepalives=1,
|
||||
keepalives_idle=60,
|
||||
keepalives_interval=10,
|
||||
keepalives_count=3,
|
||||
)
|
||||
|
||||
|
||||
def init_db_schema(cfg: AppConfig):
|
||||
with pg_connect(cfg) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
|
||||
def job_upsert_new(cur, uuid: str, jpg_key: str, meta_key: str, cat_key: Optional[str]):
|
||||
cur.execute("""
|
||||
INSERT INTO remote_cam.import_job (image_uuid, status, entrance_jpg_key, entrance_meta_key, entrance_cat_key, has_categories)
|
||||
@@ -676,15 +724,34 @@ def insert_api_response(cur, uuid: str, success: bool, message: str):
|
||||
# -----------------------
|
||||
# S3 ops wrapper
|
||||
# -----------------------
|
||||
def s3_client_config(cfg: AppConfig) -> BotoClientConfig:
|
||||
"""Explizite Timeouts/Retries fuer den Poller (botocore-Default: legacy, 60 s Read-Timeout).
|
||||
|
||||
Pfad-Adressierung + s3v4 wie im Portal; die Adressierung entspricht dem bisherigen
|
||||
Verhalten mit eigenem endpoint_url.
|
||||
"""
|
||||
return BotoClientConfig(
|
||||
max_pool_connections=cfg.s3_max_pool_connections,
|
||||
connect_timeout=cfg.s3_connect_timeout_seconds,
|
||||
read_timeout=cfg.s3_read_timeout_seconds,
|
||||
# total_max_attempts = Versuche inkl. erstem (botocores "max_attempts" zaehlt nur Wiederholungen)
|
||||
retries={"mode": cfg.s3_retry_mode, "total_max_attempts": cfg.s3_max_attempts},
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
)
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self, cfg: AppConfig):
|
||||
def __init__(self, cfg: AppConfig, client_config: Optional[BotoClientConfig] = None):
|
||||
self.cfg = cfg
|
||||
# Ohne client_config bleibt die bisherige Client-Konfiguration (so nutzt sie der
|
||||
# Gundi-Pfad); der Poller uebergibt s3_client_config(cfg).
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=cfg.s3_endpoint,
|
||||
aws_access_key_id=cfg.s3_access_key,
|
||||
aws_secret_access_key=cfg.s3_secret_key,
|
||||
config=BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
|
||||
config=client_config or BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
|
||||
)
|
||||
|
||||
def list_entrance_objects(self):
|
||||
@@ -933,15 +1000,15 @@ def cleanup_entrance_set(s3: S3, s: EntranceSet) -> int:
|
||||
return s3.delete_keys(to_delete)
|
||||
|
||||
|
||||
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
s3c = S3(cfg)
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet], s3c: S3, stats: Optional["PollerStats"] = None):
|
||||
with pg_connect(cfg) as db:
|
||||
for s in batch:
|
||||
start = time.perf_counter()
|
||||
result = "failure"
|
||||
final_status = "UNKNOWN"
|
||||
err_text = None
|
||||
cleanup_deleted = None
|
||||
abort = None
|
||||
try:
|
||||
skip_duplicate = False
|
||||
with db.transaction():
|
||||
@@ -963,6 +1030,11 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
job_set_status(cur, s.uuid, "ERROR", err=str(e))
|
||||
final_status = "ERROR"
|
||||
err_text = str(e)
|
||||
# S3/DB gestoert: Bild bleibt wie bisher auf ERROR, der Rest des Batches wird
|
||||
# nicht mehr angefasst (naechster Zyklus nach Backoff) statt jedes Bild
|
||||
# einzeln gegen die Stoerung laufen zu lassen.
|
||||
if is_transient_error(e):
|
||||
abort = e
|
||||
elapsed_ms = int((time.perf_counter() - start) * 1000)
|
||||
if err_text:
|
||||
print(
|
||||
@@ -976,19 +1048,25 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
)
|
||||
else:
|
||||
print(f"[RESULT] uuid={s.uuid} result={result} status={final_status} process_ms={elapsed_ms}")
|
||||
if stats is not None:
|
||||
stats.image_done(result == "success")
|
||||
if abort is not None:
|
||||
raise abort
|
||||
if cfg.job_sleep_seconds > 0:
|
||||
time.sleep(cfg.job_sleep_seconds)
|
||||
|
||||
|
||||
def run_once(cfg: AppConfig) -> int:
|
||||
s3c = S3(cfg)
|
||||
def run_once(cfg: AppConfig, stats: Optional["PollerStats"] = None, s3c: Optional[S3] = None) -> int:
|
||||
# Ein Client pro Zyklus fuer alle Worker: boto3-Clients sind threadsicher, das
|
||||
# parallele Anlegen ueber die Default-Session dagegen nicht.
|
||||
s3c = s3c or S3(cfg, s3_client_config(cfg))
|
||||
ready = collect_ready_sets(s3c, cfg)
|
||||
if not ready:
|
||||
return 0
|
||||
|
||||
workers = max(1, min(cfg.parallel_workers, len(ready)))
|
||||
if workers == 1:
|
||||
_process_ready_batch(cfg, ready)
|
||||
_process_ready_batch(cfg, ready, s3c, stats)
|
||||
return len(ready)
|
||||
|
||||
batches: list[list[EntranceSet]] = [[] for _ in range(workers)]
|
||||
@@ -996,13 +1074,237 @@ def run_once(cfg: AppConfig) -> int:
|
||||
batches[i % workers].append(s)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futures = [pool.submit(_process_ready_batch, cfg, batch) for batch in batches if batch]
|
||||
futures = [pool.submit(_process_ready_batch, cfg, batch, s3c, stats) for batch in batches if batch]
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
|
||||
return len(ready)
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Robustheit: Fehlerklassen, Backoff, Lebenszeichen
|
||||
# -----------------------
|
||||
TRANSIENT_S3_CODES = {
|
||||
"ServiceUnavailable", "SlowDown", "Throttling", "ThrottlingException", "TooManyRequests",
|
||||
"RequestLimitExceeded", "RequestTimeout", "InternalError", "500", "502", "503", "504",
|
||||
}
|
||||
|
||||
|
||||
def is_transient_error(exc: Optional[BaseException]) -> bool:
|
||||
"""True fuer Stoerungen, die ohne Codeaenderung vorbeigehen: S3 5xx/Throttling,
|
||||
Verbindungs-/Timeout-Fehler, DB weg (psycopg.OperationalError). Alles andere
|
||||
(4xx wie AccessDenied, Programmierfehler, SQL-Fehler) gilt als unerwartet.
|
||||
Folgt der expliziten Ursache (raise ... from ...)."""
|
||||
seen = set()
|
||||
while exc is not None and id(exc) not in seen:
|
||||
seen.add(id(exc))
|
||||
if isinstance(exc, ClientError):
|
||||
err = exc.response.get("Error") or {}
|
||||
http = (exc.response.get("ResponseMetadata") or {}).get("HTTPStatusCode") or 0
|
||||
if str(err.get("Code", "")) in TRANSIENT_S3_CODES or http >= 500 or http == 429:
|
||||
return True
|
||||
elif isinstance(exc, (BotoConnectionError, HTTPClientError, IncompleteReadError)):
|
||||
return True
|
||||
elif isinstance(exc, psycopg.OperationalError):
|
||||
return True
|
||||
elif isinstance(exc, (ConnectionError, TimeoutError)):
|
||||
return True
|
||||
exc = exc.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def describe_error(exc: BaseException, limit: int = 300) -> str:
|
||||
text = " ".join(str(exc).split())
|
||||
if len(text) > limit:
|
||||
text = text[:limit] + "..."
|
||||
return f"{type(exc).__name__}: {text}"
|
||||
|
||||
|
||||
class Backoff:
|
||||
"""Exponentiell: initial, 2x, 4x ... bis maximum; reset() nach einem erfolgreichen Zyklus."""
|
||||
|
||||
def __init__(self, initial: float, maximum: float, factor: float = 2.0):
|
||||
self.initial = initial
|
||||
self.maximum = maximum
|
||||
self.factor = factor
|
||||
self.failures = 0
|
||||
|
||||
def next_delay(self) -> float:
|
||||
# Exponent begrenzen: bei tagelanger Stoerung sonst OverflowError.
|
||||
delay = min(self.maximum, self.initial * (self.factor ** min(self.failures, 32)))
|
||||
self.failures += 1
|
||||
return delay
|
||||
|
||||
def reset(self):
|
||||
self.failures = 0
|
||||
|
||||
|
||||
def _utc_iso(ts: Optional[float]) -> str:
|
||||
if ts is None:
|
||||
return "never"
|
||||
return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
class PollerStats:
|
||||
"""Zaehler fuer Lebenszeichen-Log und Healthcheck. Threadsicher, weil die Worker je Bild melden.
|
||||
|
||||
Die Heartbeat-Datei wird bei jedem Zyklus und jedem Bild neu geschrieben; ihr mtime
|
||||
zeigt dem Docker-Healthcheck, dass die Schleife vorankommt (wartet vs. haengt)."""
|
||||
|
||||
def __init__(self, heartbeat_file: Optional[str], clock=time.time):
|
||||
self._lock = threading.Lock()
|
||||
self._clock = clock
|
||||
self.heartbeat_file = heartbeat_file
|
||||
self.started = clock()
|
||||
self.cycles = 0
|
||||
self.cycles_failed = 0
|
||||
self.images_ok = 0
|
||||
self.images_failed = 0
|
||||
self.errors_transient = 0
|
||||
self.errors_other = 0
|
||||
self.consecutive_failures = 0
|
||||
self.last_success: Optional[float] = None
|
||||
self.last_error: Optional[str] = None
|
||||
|
||||
def image_done(self, ok: bool):
|
||||
with self._lock:
|
||||
if ok:
|
||||
self.images_ok += 1
|
||||
else:
|
||||
self.images_failed += 1
|
||||
self._write_heartbeat()
|
||||
|
||||
def cycle_ok(self):
|
||||
with self._lock:
|
||||
self.cycles += 1
|
||||
self.consecutive_failures = 0
|
||||
self.last_success = self._clock()
|
||||
self._write_heartbeat()
|
||||
|
||||
def cycle_failed(self, transient: bool, message: str):
|
||||
with self._lock:
|
||||
self.cycles += 1
|
||||
self.cycles_failed += 1
|
||||
self.consecutive_failures += 1
|
||||
if transient:
|
||||
self.errors_transient += 1
|
||||
else:
|
||||
self.errors_other += 1
|
||||
self.last_error = message
|
||||
self._write_heartbeat()
|
||||
|
||||
def touch(self):
|
||||
with self._lock:
|
||||
self._write_heartbeat()
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
now = self._clock()
|
||||
return {
|
||||
"ts": _utc_iso(now),
|
||||
"uptime_s": int(now - self.started),
|
||||
"cycles": self.cycles,
|
||||
"cycles_failed": self.cycles_failed,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"images_ok": self.images_ok,
|
||||
"images_failed": self.images_failed,
|
||||
"errors_transient": self.errors_transient,
|
||||
"errors_other": self.errors_other,
|
||||
"last_success": _utc_iso(self.last_success),
|
||||
"last_success_age_s": None if self.last_success is None else int(now - self.last_success),
|
||||
"last_error": self.last_error,
|
||||
}
|
||||
|
||||
def summary(self) -> str:
|
||||
s = self.snapshot()
|
||||
age = "-" if s["last_success_age_s"] is None else f"{s['last_success_age_s']}s"
|
||||
return (
|
||||
f"uptime={s['uptime_s']}s cycles={s['cycles']} cycles_failed={s['cycles_failed']} "
|
||||
f"consecutive_failures={s['consecutive_failures']} images_ok={s['images_ok']} "
|
||||
f"images_failed={s['images_failed']} errors_transient={s['errors_transient']} "
|
||||
f"errors_other={s['errors_other']} last_success={s['last_success']} last_success_age={age}"
|
||||
)
|
||||
|
||||
def _write_heartbeat(self):
|
||||
if not self.heartbeat_file:
|
||||
return
|
||||
try:
|
||||
tmp = f"{self.heartbeat_file}.tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(self.snapshot(), f)
|
||||
os.replace(tmp, self.heartbeat_file)
|
||||
except OSError as e:
|
||||
print(f"[WARN] heartbeat file not writable: {e}")
|
||||
|
||||
|
||||
def poll_forever(
|
||||
cfg: AppConfig,
|
||||
*,
|
||||
cycle=run_once,
|
||||
init_schema=init_db_schema,
|
||||
sleep=time.sleep,
|
||||
clock=time.time,
|
||||
stats: Optional[PollerStats] = None,
|
||||
max_iterations: Optional[int] = None,
|
||||
) -> PollerStats:
|
||||
"""Hauptschleife. Faengt Fehler je Zyklus ab und wartet mit Backoff - S3- oder DB-Ausfaelle
|
||||
beenden den Prozess nicht mehr. db_init laeuft einmal erfolgreich, danach nicht erneut.
|
||||
Transaktion pro Bild und Statusfolge liegen unveraendert in _process_ready_batch/process_one.
|
||||
"""
|
||||
stats = stats or PollerStats(cfg.heartbeat_file, clock=clock)
|
||||
backoff = Backoff(cfg.backoff_initial_seconds, cfg.backoff_max_seconds)
|
||||
schema_ready = False
|
||||
failing_since: Optional[float] = None
|
||||
last_heartbeat_log = clock()
|
||||
iterations = 0
|
||||
|
||||
while max_iterations is None or iterations < max_iterations:
|
||||
iterations += 1
|
||||
stats.touch()
|
||||
ok_before = stats.images_ok
|
||||
try:
|
||||
if not schema_ready:
|
||||
init_schema(cfg)
|
||||
schema_ready = True
|
||||
n = cycle(cfg, stats)
|
||||
except Exception as e:
|
||||
transient = is_transient_error(e)
|
||||
msg = describe_error(e)
|
||||
delay = backoff.next_delay()
|
||||
stats.cycle_failed(transient, msg)
|
||||
if failing_since is None:
|
||||
failing_since = clock()
|
||||
kind = "transient" if transient else "unexpected"
|
||||
print(
|
||||
f"[CYCLE_FAIL] kind={kind} consecutive={backoff.failures} "
|
||||
f"retry_in={delay:.0f}s error={msg}"
|
||||
)
|
||||
if not transient:
|
||||
traceback.print_exc()
|
||||
n = None
|
||||
else:
|
||||
if failing_since is not None:
|
||||
print(
|
||||
f"[RECOVERED] after_failures={backoff.failures} "
|
||||
f"outage_s={int(clock() - failing_since)}"
|
||||
)
|
||||
failing_since = None
|
||||
backoff.reset()
|
||||
stats.cycle_ok()
|
||||
delay = None
|
||||
|
||||
if clock() - last_heartbeat_log >= cfg.heartbeat_seconds:
|
||||
print(f"[HEARTBEAT] {stats.summary()}")
|
||||
last_heartbeat_log = clock()
|
||||
|
||||
if delay is not None:
|
||||
sleep(delay)
|
||||
elif n == 0 or stats.images_ok == ok_before:
|
||||
# Nichts zu tun - oder nur Saetze, die erneut scheitern: nicht im Leerlauf kreiseln.
|
||||
sleep(cfg.poll_seconds)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
if os.environ.get("DEBUG") == "1":
|
||||
import debugpy
|
||||
@@ -1013,15 +1315,13 @@ def main():
|
||||
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
|
||||
cfg = load_config(cfg_path)
|
||||
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
while True:
|
||||
n = run_once(cfg)
|
||||
if n == 0:
|
||||
time.sleep(cfg.poll_seconds)
|
||||
print(
|
||||
f"[START] poll={cfg.poll_seconds}s workers={cfg.parallel_workers} "
|
||||
f"s3_timeout=connect:{cfg.s3_connect_timeout_seconds:g}s/read:{cfg.s3_read_timeout_seconds:g}s "
|
||||
f"s3_retries={cfg.s3_retry_mode}/{cfg.s3_max_attempts} pg_connect_timeout={cfg.pg_connect_timeout_seconds}s "
|
||||
f"backoff={cfg.backoff_initial_seconds:g}-{cfg.backoff_max_seconds:g}s heartbeat={cfg.heartbeat_seconds}s"
|
||||
)
|
||||
poll_forever(cfg)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user