diff --git a/README.md b/README.md index 2e35303..0b8357a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,45 @@ docker compose up -d --build docker compose logs -f ``` +### Stoerungen und Lebenszeichen (Poller) + +Fehler eines Zyklus (S3 5xx/Throttling, Netz, DB weg) beenden den Poller nicht: +er loggt eine Zeile `[CYCLE_FAIL] kind=transient ...` und versucht es mit +exponentiellem Backoff erneut (`backoff_initial_seconds` bis `backoff_max_seconds`, +Reset nach Erfolg, dann `[RECOVERED]`). Unerwartete Fehler (`kind=unexpected`, +z. B. AccessDenied oder Programmierfehler) werden zusaetzlich mit Traceback geloggt. +Ein Bild, bei dem S3/DB wegbricht, bleibt wie bisher auf `ERROR`; der Rest des +Batches wartet auf den naechsten Zyklus. + +Alle `heartbeat_seconds` erscheint `[HEARTBEAT]` mit Zaehlern (Zyklen, Bilder, +Fehler, letzter Erfolg). Der Docker-Healthcheck prueft, ob die Hauptschleife +vorankommt (`/tmp/melesicumover.heartbeat` juenger als 15 min): + +```bash +docker inspect -f '{{.State.Health.Status}} restarts={{.RestartCount}}' melesicumover +docker exec melesicumover cat /tmp/melesicumover.heartbeat +``` + +S3-Timeouts/-Retries stehen unter `app.s3_*` in `config.yaml`. + +### Eingang pruefen (verwaiste Objekte) + +```bash +docker compose run --rm --no-deps melesicumover python -m tools.entrance_inventory --db +``` + +Listet `icu/entrance/` nach Typ (vollstaendig, JPG ohne JSON, JSON ohne JPG, +nur Kategorien, fremd) und Alter, optional mit `import_job`-Status. Nur lesend. + +### Tests + +```bash +pip install -r requirements.txt +python -m unittest discover -s tests -v # oder: python -m pytest tests +``` + +Simulierte S3-/DB-Stoerungen fuer Backoff und Fehlerbehandlung; weder Netz noch DB noetig. + ## Gundi-Push-Pfad (`gundi-receiver`) ICU pusht Events ueber eine Gundi-v2-kompatible API. Der Service `gundi-receiver` diff --git a/config.yaml b/config.yaml index e84758d..f93a923 100644 --- a/config.yaml +++ b/config.yaml @@ -39,6 +39,15 @@ app: enable_deermapper_api: false deermapper_api_timeout_seconds: 20 deermapper_api_image_field: "image" + # Robustheit (nur Poller): S3-Timeouts/Retries, Backoff bei Stoerungen, Lebenszeichen + s3_connect_timeout_seconds: 10 + s3_read_timeout_seconds: 30 + s3_retry_mode: "standard" # standard | adaptive | legacy + s3_max_attempts: 4 # inkl. erstem Versuch + pg_connect_timeout_seconds: 10 + backoff_initial_seconds: 5 # 5 s, 10 s, 20 s ... nach fehlgeschlagenem Zyklus + backoff_max_seconds: 300 # hoechstens 5 min; Reset nach erfolgreichem Zyklus + heartbeat_seconds: 300 # [HEARTBEAT]-Logzeile mit Zaehlern # Gundi-v2-Push-Empfaenger (Service gundi-receiver). # api key -> via ENV (.env): GUNDI_API_KEY (min. 24 Zeichen) diff --git a/docker-compose.yaml b/docker-compose.yaml index 2876bda..5213ea1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,6 +10,14 @@ services: environment: CONFIG_YAML: /app/config.yaml DEBUG: "0" + # Lebenszeichen: die Hauptschleife schreibt die Heartbeat-Datei bei jedem Zyklus + # und jedem Bild. Laenger als 15 min unveraendert = Schleife haengt (Backoff max. 5 min). + healthcheck: + test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/melesicumover.heartbeat') < 900 else 1)"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 120s # Kein Bind-Mount des Quellcodes mehr (der Code steckt im Image) und # standardmaessig KEIN offener Debug-Port. # diff --git a/main.py b/main.py index e7504e9..9e30955 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,8 @@ import json import time import tempfile import subprocess +import threading +import traceback import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass @@ -17,6 +19,12 @@ from urllib import request as urlrequest import boto3 import psycopg from botocore.config import Config as BotoClientConfig +from botocore.exceptions import ( + ClientError, + ConnectionError as BotoConnectionError, + HTTPClientError, + IncompleteReadError, +) from dateutil import tz from PIL import Image, ImageOps, ImageEnhance import pytesseract @@ -73,6 +81,17 @@ class AppConfig: deermapper_api_timeout_seconds: int = 20 deermapper_api_image_field: str = "image" + # Robustheit Poller: S3-Timeouts/Retries, DB-Connect-Timeout, Backoff, Lebenszeichen + s3_connect_timeout_seconds: float = 10.0 + s3_read_timeout_seconds: float = 30.0 + s3_retry_mode: str = "standard" + s3_max_attempts: int = 4 + pg_connect_timeout_seconds: int = 10 + backoff_initial_seconds: float = 5.0 + backoff_max_seconds: float = 300.0 + heartbeat_seconds: int = 300 + heartbeat_file: str = "/tmp/melesicumover.heartbeat" + def load_config(path: str) -> AppConfig: with open(path, "r", encoding="utf-8") as f: @@ -128,6 +147,15 @@ def load_config(path: str) -> AppConfig: deermapper_api_key=env_or("DEERMAPPER_API_KEY", deermapper.get("apiKey", "")), deermapper_api_timeout_seconds=int(app.get("deermapper_api_timeout_seconds", 20)), deermapper_api_image_field=app.get("deermapper_api_image_field", "image"), + s3_connect_timeout_seconds=float(app.get("s3_connect_timeout_seconds", 10)), + s3_read_timeout_seconds=float(app.get("s3_read_timeout_seconds", 30)), + s3_retry_mode=str(app.get("s3_retry_mode", "standard")), + s3_max_attempts=max(1, int(app.get("s3_max_attempts", 4))), + pg_connect_timeout_seconds=max(1, int(app.get("pg_connect_timeout_seconds", 10))), + backoff_initial_seconds=max(0.1, float(app.get("backoff_initial_seconds", 5))), + backoff_max_seconds=max(1.0, float(app.get("backoff_max_seconds", 300))), + heartbeat_seconds=max(10, int(app.get("heartbeat_seconds", 300))), + heartbeat_file=app.get("heartbeat_file", "/tmp/melesicumover.heartbeat"), ) # Fail-fast: Secrets kommen ausschliesslich aus der Umgebung (.env). Fehlen sie, @@ -454,6 +482,26 @@ def db_init(cur): """) +def pg_connect(cfg: AppConfig) -> psycopg.Connection: + # connect_timeout + TCP-Keepalives: ein weggebrochener DB-Host blockiert den + # Poller nicht unbegrenzt, sondern endet als OperationalError (-> Backoff). + return psycopg.connect( + cfg.pg_dsn, + connect_timeout=cfg.pg_connect_timeout_seconds, + keepalives=1, + keepalives_idle=60, + keepalives_interval=10, + keepalives_count=3, + ) + + +def init_db_schema(cfg: AppConfig): + with pg_connect(cfg) as db: + with db.cursor() as cur: + db_init(cur) + db.commit() + + def job_upsert_new(cur, uuid: str, jpg_key: str, meta_key: str, cat_key: Optional[str]): cur.execute(""" INSERT INTO remote_cam.import_job (image_uuid, status, entrance_jpg_key, entrance_meta_key, entrance_cat_key, has_categories) @@ -676,15 +724,34 @@ def insert_api_response(cur, uuid: str, success: bool, message: str): # ----------------------- # S3 ops wrapper # ----------------------- +def s3_client_config(cfg: AppConfig) -> BotoClientConfig: + """Explizite Timeouts/Retries fuer den Poller (botocore-Default: legacy, 60 s Read-Timeout). + + Pfad-Adressierung + s3v4 wie im Portal; die Adressierung entspricht dem bisherigen + Verhalten mit eigenem endpoint_url. + """ + return BotoClientConfig( + max_pool_connections=cfg.s3_max_pool_connections, + connect_timeout=cfg.s3_connect_timeout_seconds, + read_timeout=cfg.s3_read_timeout_seconds, + # total_max_attempts = Versuche inkl. erstem (botocores "max_attempts" zaehlt nur Wiederholungen) + retries={"mode": cfg.s3_retry_mode, "total_max_attempts": cfg.s3_max_attempts}, + signature_version="s3v4", + s3={"addressing_style": "path"}, + ) + + class S3: - def __init__(self, cfg: AppConfig): + def __init__(self, cfg: AppConfig, client_config: Optional[BotoClientConfig] = None): self.cfg = cfg + # Ohne client_config bleibt die bisherige Client-Konfiguration (so nutzt sie der + # Gundi-Pfad); der Poller uebergibt s3_client_config(cfg). self.client = boto3.client( "s3", endpoint_url=cfg.s3_endpoint, aws_access_key_id=cfg.s3_access_key, aws_secret_access_key=cfg.s3_secret_key, - config=BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections), + config=client_config or BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections), ) def list_entrance_objects(self): @@ -933,15 +1000,15 @@ def cleanup_entrance_set(s3: S3, s: EntranceSet) -> int: return s3.delete_keys(to_delete) -def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]): - s3c = S3(cfg) - with psycopg.connect(cfg.pg_dsn) as db: +def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet], s3c: S3, stats: Optional["PollerStats"] = None): + with pg_connect(cfg) as db: for s in batch: start = time.perf_counter() result = "failure" final_status = "UNKNOWN" err_text = None cleanup_deleted = None + abort = None try: skip_duplicate = False with db.transaction(): @@ -963,6 +1030,11 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]): job_set_status(cur, s.uuid, "ERROR", err=str(e)) final_status = "ERROR" err_text = str(e) + # S3/DB gestoert: Bild bleibt wie bisher auf ERROR, der Rest des Batches wird + # nicht mehr angefasst (naechster Zyklus nach Backoff) statt jedes Bild + # einzeln gegen die Stoerung laufen zu lassen. + if is_transient_error(e): + abort = e elapsed_ms = int((time.perf_counter() - start) * 1000) if err_text: print( @@ -976,19 +1048,25 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]): ) else: print(f"[RESULT] uuid={s.uuid} result={result} status={final_status} process_ms={elapsed_ms}") + if stats is not None: + stats.image_done(result == "success") + if abort is not None: + raise abort if cfg.job_sleep_seconds > 0: time.sleep(cfg.job_sleep_seconds) -def run_once(cfg: AppConfig) -> int: - s3c = S3(cfg) +def run_once(cfg: AppConfig, stats: Optional["PollerStats"] = None, s3c: Optional[S3] = None) -> int: + # Ein Client pro Zyklus fuer alle Worker: boto3-Clients sind threadsicher, das + # parallele Anlegen ueber die Default-Session dagegen nicht. + s3c = s3c or S3(cfg, s3_client_config(cfg)) ready = collect_ready_sets(s3c, cfg) if not ready: return 0 workers = max(1, min(cfg.parallel_workers, len(ready))) if workers == 1: - _process_ready_batch(cfg, ready) + _process_ready_batch(cfg, ready, s3c, stats) return len(ready) batches: list[list[EntranceSet]] = [[] for _ in range(workers)] @@ -996,13 +1074,237 @@ def run_once(cfg: AppConfig) -> int: batches[i % workers].append(s) with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(_process_ready_batch, cfg, batch) for batch in batches if batch] + futures = [pool.submit(_process_ready_batch, cfg, batch, s3c, stats) for batch in batches if batch] for future in as_completed(futures): future.result() return len(ready) +# ----------------------- +# Robustheit: Fehlerklassen, Backoff, Lebenszeichen +# ----------------------- +TRANSIENT_S3_CODES = { + "ServiceUnavailable", "SlowDown", "Throttling", "ThrottlingException", "TooManyRequests", + "RequestLimitExceeded", "RequestTimeout", "InternalError", "500", "502", "503", "504", +} + + +def is_transient_error(exc: Optional[BaseException]) -> bool: + """True fuer Stoerungen, die ohne Codeaenderung vorbeigehen: S3 5xx/Throttling, + Verbindungs-/Timeout-Fehler, DB weg (psycopg.OperationalError). Alles andere + (4xx wie AccessDenied, Programmierfehler, SQL-Fehler) gilt als unerwartet. + Folgt der expliziten Ursache (raise ... from ...).""" + seen = set() + while exc is not None and id(exc) not in seen: + seen.add(id(exc)) + if isinstance(exc, ClientError): + err = exc.response.get("Error") or {} + http = (exc.response.get("ResponseMetadata") or {}).get("HTTPStatusCode") or 0 + if str(err.get("Code", "")) in TRANSIENT_S3_CODES or http >= 500 or http == 429: + return True + elif isinstance(exc, (BotoConnectionError, HTTPClientError, IncompleteReadError)): + return True + elif isinstance(exc, psycopg.OperationalError): + return True + elif isinstance(exc, (ConnectionError, TimeoutError)): + return True + exc = exc.__cause__ + return False + + +def describe_error(exc: BaseException, limit: int = 300) -> str: + text = " ".join(str(exc).split()) + if len(text) > limit: + text = text[:limit] + "..." + return f"{type(exc).__name__}: {text}" + + +class Backoff: + """Exponentiell: initial, 2x, 4x ... bis maximum; reset() nach einem erfolgreichen Zyklus.""" + + def __init__(self, initial: float, maximum: float, factor: float = 2.0): + self.initial = initial + self.maximum = maximum + self.factor = factor + self.failures = 0 + + def next_delay(self) -> float: + # Exponent begrenzen: bei tagelanger Stoerung sonst OverflowError. + delay = min(self.maximum, self.initial * (self.factor ** min(self.failures, 32))) + self.failures += 1 + return delay + + def reset(self): + self.failures = 0 + + +def _utc_iso(ts: Optional[float]) -> str: + if ts is None: + return "never" + return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class PollerStats: + """Zaehler fuer Lebenszeichen-Log und Healthcheck. Threadsicher, weil die Worker je Bild melden. + + Die Heartbeat-Datei wird bei jedem Zyklus und jedem Bild neu geschrieben; ihr mtime + zeigt dem Docker-Healthcheck, dass die Schleife vorankommt (wartet vs. haengt).""" + + def __init__(self, heartbeat_file: Optional[str], clock=time.time): + self._lock = threading.Lock() + self._clock = clock + self.heartbeat_file = heartbeat_file + self.started = clock() + self.cycles = 0 + self.cycles_failed = 0 + self.images_ok = 0 + self.images_failed = 0 + self.errors_transient = 0 + self.errors_other = 0 + self.consecutive_failures = 0 + self.last_success: Optional[float] = None + self.last_error: Optional[str] = None + + def image_done(self, ok: bool): + with self._lock: + if ok: + self.images_ok += 1 + else: + self.images_failed += 1 + self._write_heartbeat() + + def cycle_ok(self): + with self._lock: + self.cycles += 1 + self.consecutive_failures = 0 + self.last_success = self._clock() + self._write_heartbeat() + + def cycle_failed(self, transient: bool, message: str): + with self._lock: + self.cycles += 1 + self.cycles_failed += 1 + self.consecutive_failures += 1 + if transient: + self.errors_transient += 1 + else: + self.errors_other += 1 + self.last_error = message + self._write_heartbeat() + + def touch(self): + with self._lock: + self._write_heartbeat() + + def snapshot(self) -> dict: + now = self._clock() + return { + "ts": _utc_iso(now), + "uptime_s": int(now - self.started), + "cycles": self.cycles, + "cycles_failed": self.cycles_failed, + "consecutive_failures": self.consecutive_failures, + "images_ok": self.images_ok, + "images_failed": self.images_failed, + "errors_transient": self.errors_transient, + "errors_other": self.errors_other, + "last_success": _utc_iso(self.last_success), + "last_success_age_s": None if self.last_success is None else int(now - self.last_success), + "last_error": self.last_error, + } + + def summary(self) -> str: + s = self.snapshot() + age = "-" if s["last_success_age_s"] is None else f"{s['last_success_age_s']}s" + return ( + f"uptime={s['uptime_s']}s cycles={s['cycles']} cycles_failed={s['cycles_failed']} " + f"consecutive_failures={s['consecutive_failures']} images_ok={s['images_ok']} " + f"images_failed={s['images_failed']} errors_transient={s['errors_transient']} " + f"errors_other={s['errors_other']} last_success={s['last_success']} last_success_age={age}" + ) + + def _write_heartbeat(self): + if not self.heartbeat_file: + return + try: + tmp = f"{self.heartbeat_file}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(self.snapshot(), f) + os.replace(tmp, self.heartbeat_file) + except OSError as e: + print(f"[WARN] heartbeat file not writable: {e}") + + +def poll_forever( + cfg: AppConfig, + *, + cycle=run_once, + init_schema=init_db_schema, + sleep=time.sleep, + clock=time.time, + stats: Optional[PollerStats] = None, + max_iterations: Optional[int] = None, +) -> PollerStats: + """Hauptschleife. Faengt Fehler je Zyklus ab und wartet mit Backoff - S3- oder DB-Ausfaelle + beenden den Prozess nicht mehr. db_init laeuft einmal erfolgreich, danach nicht erneut. + Transaktion pro Bild und Statusfolge liegen unveraendert in _process_ready_batch/process_one. + """ + stats = stats or PollerStats(cfg.heartbeat_file, clock=clock) + backoff = Backoff(cfg.backoff_initial_seconds, cfg.backoff_max_seconds) + schema_ready = False + failing_since: Optional[float] = None + last_heartbeat_log = clock() + iterations = 0 + + while max_iterations is None or iterations < max_iterations: + iterations += 1 + stats.touch() + ok_before = stats.images_ok + try: + if not schema_ready: + init_schema(cfg) + schema_ready = True + n = cycle(cfg, stats) + except Exception as e: + transient = is_transient_error(e) + msg = describe_error(e) + delay = backoff.next_delay() + stats.cycle_failed(transient, msg) + if failing_since is None: + failing_since = clock() + kind = "transient" if transient else "unexpected" + print( + f"[CYCLE_FAIL] kind={kind} consecutive={backoff.failures} " + f"retry_in={delay:.0f}s error={msg}" + ) + if not transient: + traceback.print_exc() + n = None + else: + if failing_since is not None: + print( + f"[RECOVERED] after_failures={backoff.failures} " + f"outage_s={int(clock() - failing_since)}" + ) + failing_since = None + backoff.reset() + stats.cycle_ok() + delay = None + + if clock() - last_heartbeat_log >= cfg.heartbeat_seconds: + print(f"[HEARTBEAT] {stats.summary()}") + last_heartbeat_log = clock() + + if delay is not None: + sleep(delay) + elif n == 0 or stats.images_ok == ok_before: + # Nichts zu tun - oder nur Saetze, die erneut scheitern: nicht im Leerlauf kreiseln. + sleep(cfg.poll_seconds) + + return stats + + def main(): if os.environ.get("DEBUG") == "1": import debugpy @@ -1013,15 +1315,13 @@ def main(): cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml") cfg = load_config(cfg_path) - with psycopg.connect(cfg.pg_dsn) as db: - with db.cursor() as cur: - db_init(cur) - db.commit() - - while True: - n = run_once(cfg) - if n == 0: - time.sleep(cfg.poll_seconds) + print( + f"[START] poll={cfg.poll_seconds}s workers={cfg.parallel_workers} " + f"s3_timeout=connect:{cfg.s3_connect_timeout_seconds:g}s/read:{cfg.s3_read_timeout_seconds:g}s " + f"s3_retries={cfg.s3_retry_mode}/{cfg.s3_max_attempts} pg_connect_timeout={cfg.pg_connect_timeout_seconds}s " + f"backoff={cfg.backoff_initial_seconds:g}-{cfg.backoff_max_seconds:g}s heartbeat={cfg.heartbeat_seconds}s" + ) + poll_forever(cfg) if __name__ == "__main__": diff --git a/tests/test_poller_resilience.py b/tests/test_poller_resilience.py new file mode 100644 index 0000000..f68e1ab --- /dev/null +++ b/tests/test_poller_resilience.py @@ -0,0 +1,440 @@ +"""Backoff und Fehlerbehandlung des Legacy-Pollers mit simulierten S3-/DB-Stoerungen. + + python -m unittest discover -s tests -v (oder: python -m pytest tests) + +Braucht die Pakete aus requirements.txt, aber weder Netz noch DB. +""" +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import psycopg # noqa: E402 +from botocore.exceptions import ( # noqa: E402 + ClientError, + ConnectTimeoutError, + ConnectionClosedError, + EndpointConnectionError, + ReadTimeoutError, +) + +import main # noqa: E402 + + +def client_error(code: str, http: int, op: str = "ListObjectsV2") -> ClientError: + return ClientError({"Error": {"Code": code, "Message": code}, "ResponseMetadata": {"HTTPStatusCode": http}}, op) + + +def s3_unavailable() -> ClientError: + return client_error("ServiceUnavailable", 503) + + +def make_cfg(tmpdir: str, **overrides) -> main.AppConfig: + params = dict( + s3_endpoint="https://s3.invalid/", + s3_access_key="test", + s3_secret_key="test", + s3_bucket="bucket", + pg_dsn="postgresql://invalid/db", + poll_seconds=30, + parallel_workers=1, + backoff_initial_seconds=5, + backoff_max_seconds=300, + heartbeat_seconds=300, + heartbeat_file=os.path.join(tmpdir, "heartbeat.json"), + ) + params.update(overrides) + return main.AppConfig(**params) + + +class FakeClock: + """Zeit laeuft nur durch sleep() - Tests laufen ohne echtes Warten.""" + + def __init__(self, start: float = 1_000_000.0): + self.now = start + self.sleeps: list[float] = [] + + def time(self) -> float: + return self.now + + def sleep(self, seconds: float): + self.sleeps.append(seconds) + self.now += seconds + + +class ScriptedCycle: + """Zyklus-Ersatz: arbeitet eine Liste ab (Exception -> werfen, int -> zurueckgeben).""" + + def __init__(self, steps): + self.steps = list(steps) + self.calls = 0 + + def __call__(self, cfg, stats): + self.calls += 1 + step = self.steps.pop(0) if self.steps else 0 + if isinstance(step, BaseException): + raise step + if callable(step): + return step(stats) + return step + + +class FakeS3: + """Simulierter S3-Client: Listing liefert Objekte oder wirft den angegebenen Fehler.""" + + def __init__(self, objects=None, list_error: BaseException = None): + self.objects = objects or [] + self.list_error = list_error + + def list_entrance_objects(self): + if self.list_error is not None: + raise self.list_error + yield from self.objects + + +class FakeConn: + """Simulierte psycopg-Verbindung; die SQL-Funktionen werden in den Tests gepatcht.""" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + @contextlib.contextmanager + def transaction(self): + yield + + def cursor(self): + return contextlib.nullcontext(object()) + + +def entrance_objects(*uuids): + old = datetime.now(timezone.utc) - timedelta(hours=1) + objs = [] + for u in uuids: + objs.append({"Key": f"icu/entrance/{u}.jpg", "LastModified": old}) + objs.append({"Key": f"icu/entrance/{u}.json", "LastModified": old}) + return objs + + +UUIDS = [f"00000000-0000-4000-8000-00000000000{i}" for i in range(1, 4)] + + +class BackoffTest(unittest.TestCase): + def test_doubles_up_to_maximum(self): + b = main.Backoff(5, 300) + self.assertEqual([b.next_delay() for _ in range(9)], [5, 10, 20, 40, 80, 160, 300, 300, 300]) + + def test_reset_starts_again_at_initial(self): + b = main.Backoff(5, 300) + for _ in range(4): + b.next_delay() + b.reset() + self.assertEqual(b.failures, 0) + self.assertEqual(b.next_delay(), 5) + + def test_long_outage_does_not_overflow(self): + b = main.Backoff(5, 300) + for _ in range(5000): + delay = b.next_delay() + self.assertEqual(delay, 300) + + +class ErrorClassificationTest(unittest.TestCase): + def test_transient(self): + cases = [ + s3_unavailable(), + client_error("SlowDown", 503), + client_error("InternalError", 500), + client_error("TooManyRequests", 429), + client_error("Whatever", 502), + EndpointConnectionError(endpoint_url="https://s3.invalid"), + ConnectTimeoutError(endpoint_url="https://s3.invalid"), + ReadTimeoutError(endpoint_url="https://s3.invalid"), + ConnectionClosedError(endpoint_url="https://s3.invalid"), + psycopg.OperationalError("connection refused"), + psycopg.errors.AdminShutdown("terminating connection due to administrator command"), + ConnectionResetError(), + TimeoutError(), + ] + for exc in cases: + with self.subTest(exc=repr(exc)): + self.assertTrue(main.is_transient_error(exc)) + + def test_unexpected(self): + cases = [ + client_error("AccessDenied", 403), + client_error("NoSuchKey", 404, "GetObject"), + client_error("NoSuchBucket", 404), + psycopg.ProgrammingError("syntax error"), + psycopg.errors.InsufficientPrivilege("permission denied"), + KeyError("status"), + ValueError("metadata.servertime missing"), + ] + for exc in cases: + with self.subTest(exc=repr(exc)): + self.assertFalse(main.is_transient_error(exc)) + + def test_follows_explicit_cause_only(self): + try: + try: + raise s3_unavailable() + except ClientError as e: + raise RuntimeError("wrapped") from e + except RuntimeError as wrapped: + self.assertTrue(main.is_transient_error(wrapped)) + + # Ein Bug im Fehlerpfad (nur __context__) bleibt ein unerwarteter Fehler. + try: + try: + raise s3_unavailable() + except ClientError: + raise KeyError("bug in handler") + except KeyError as bug: + self.assertFalse(main.is_transient_error(bug)) + + +class PollForeverTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cfg = make_cfg(self.tmp.name) + self.clock = FakeClock() + self.out = io.StringIO() + self.err = io.StringIO() + + def tearDown(self): + self.tmp.cleanup() + + def run_loop(self, cycle, iterations, init_schema=lambda cfg: None): + with contextlib.redirect_stdout(self.out), contextlib.redirect_stderr(self.err): + return main.poll_forever( + self.cfg, + cycle=cycle, + init_schema=init_schema, + sleep=self.clock.sleep, + clock=self.clock.time, + max_iterations=iterations, + ) + + def test_s3_outage_backs_off_and_recovers(self): + cycle = ScriptedCycle([s3_unavailable(), s3_unavailable(), s3_unavailable(), 0, 0, s3_unavailable(), 0]) + stats = self.run_loop(cycle, 7) + self.assertEqual(self.clock.sleeps, [5, 10, 20, 30, 30, 5, 30]) + self.assertEqual(stats.cycles, 7) + self.assertEqual(stats.cycles_failed, 4) + self.assertEqual(stats.errors_transient, 4) + self.assertEqual(stats.errors_other, 0) + self.assertEqual(stats.consecutive_failures, 0) + self.assertIsNotNone(stats.last_success) + log = self.out.getvalue() + self.assertIn("[CYCLE_FAIL] kind=transient consecutive=1 retry_in=5s error=ClientError", log) + self.assertIn("[RECOVERED] after_failures=3 outage_s=35", log) + self.assertNotIn("Traceback", self.err.getvalue()) + + def test_backoff_is_capped_during_long_outage(self): + stats = self.run_loop(ScriptedCycle([s3_unavailable()] * 12), 12) + self.assertEqual(self.clock.sleeps[-4:], [300, 300, 300, 300]) + self.assertEqual(stats.consecutive_failures, 12) + + def test_db_unavailable_at_start_is_retried_then_schema_init_runs_once(self): + calls = [] + + def init_schema(cfg): + calls.append(1) + if len(calls) <= 2: + raise psycopg.OperationalError("could not connect to server") + + cycle = ScriptedCycle([0, 0, 0]) + stats = self.run_loop(cycle, 5, init_schema=init_schema) + self.assertEqual(len(calls), 3) + self.assertEqual(cycle.calls, 3) + self.assertEqual(self.clock.sleeps, [5, 10, 30, 30, 30]) + self.assertEqual(stats.errors_transient, 2) + + def test_unexpected_error_is_logged_with_traceback_but_does_not_crash(self): + cycle = ScriptedCycle([KeyError("boom"), KeyError("boom"), 0]) + stats = self.run_loop(cycle, 3) + self.assertEqual(stats.errors_other, 2) + self.assertEqual(self.clock.sleeps, [5, 10, 30]) + self.assertIn("[CYCLE_FAIL] kind=unexpected", self.out.getvalue()) + self.assertIn("Traceback", self.err.getvalue()) + self.assertIn("KeyError: 'boom'", self.err.getvalue()) + + def test_no_busy_loop_when_ready_sets_keep_failing(self): + def progress(stats): + stats.image_done(True) + return 1 + + cycle = ScriptedCycle([2, progress, 0]) + self.run_loop(cycle, 3) + # 2 Saetze ohne Erfolg -> Poll-Pause; Fortschritt -> sofort weiter; leer -> Poll-Pause + self.assertEqual(self.clock.sleeps, [30, 30]) + + def test_heartbeat_log_and_file(self): + stats = self.run_loop(ScriptedCycle([0] * 12), 12) + beats = [line for line in self.out.getvalue().splitlines() if line.startswith("[HEARTBEAT]")] + self.assertEqual(len(beats), 1) # Zyklus 11 laeuft bei +300 s -> genau eine Zeile + self.assertIn("cycles=11 cycles_failed=0", beats[0]) + self.assertIn("last_success_age=0s", beats[0]) + with open(self.cfg.heartbeat_file, encoding="utf-8") as f: + data = json.load(f) + self.assertEqual(data["cycles"], stats.cycles) + self.assertEqual(data["errors_transient"], 0) + + +class RunOnceWithFakeS3Test(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cfg = make_cfg(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_listing_failure_propagates_to_loop_which_survives(self): + s3 = FakeS3(list_error=s3_unavailable()) + with self.assertRaises(ClientError): + main.run_once(self.cfg, None, s3c=s3) + + clock = FakeClock() + with contextlib.redirect_stdout(io.StringIO()): + stats = main.poll_forever( + self.cfg, + cycle=lambda cfg, st: main.run_once(cfg, st, s3c=s3), + init_schema=lambda cfg: None, + sleep=clock.sleep, + clock=clock.time, + max_iterations=3, + ) + self.assertEqual(clock.sleeps, [5, 10, 20]) + self.assertEqual(stats.errors_transient, 3) + + def test_empty_entrance_returns_zero(self): + self.assertEqual(main.run_once(self.cfg, None, s3c=FakeS3()), 0) + + +class ProcessBatchTest(unittest.TestCase): + """Transaktion pro Bild bleibt; bei S3-/DB-Stoerung wird der Rest des Batches nicht angefasst.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cfg = make_cfg(self.tmp.name) + self.status_calls = [] + self.processed = [] + patches = [ + mock.patch.object(main, "pg_connect", lambda cfg: FakeConn()), + mock.patch.object(main, "is_completed_job_materialized", lambda cur, s3, cfg, uuid: False), + mock.patch.object(main, "job_upsert_new", lambda cur, uuid, *a: None), + mock.patch.object(main, "job_get", lambda cur, uuid: {"status": "CLEANED"}), + mock.patch.object( + main, "job_set_status", + lambda cur, uuid, status, **kw: self.status_calls.append((uuid, status, kw.get("err"))), + ), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def tearDown(self): + self.tmp.cleanup() + + def batch(self): + s3 = FakeS3(entrance_objects(*UUIDS)) + return main.collect_ready_sets(s3, self.cfg), s3 + + def fail_first_with(self, exc): + def process_one(cur, s3, cfg, s): + self.processed.append(s.uuid) + if s.uuid == UUIDS[0]: + raise exc + return mock.patch.object(main, "process_one", process_one) + + def test_transient_s3_error_marks_image_error_and_aborts_batch(self): + sets, s3 = self.batch() + stats = main.PollerStats(None) + with self.fail_first_with(client_error("ServiceUnavailable", 503, "GetObject")), \ + contextlib.redirect_stdout(io.StringIO()) as out: + with self.assertRaises(ClientError): + main._process_ready_batch(self.cfg, sets, s3, stats) + self.assertEqual(self.processed, [UUIDS[0]]) + self.assertEqual([(u, st) for u, st, _ in self.status_calls], [(UUIDS[0], "ERROR")]) + self.assertIn(f"[RESULT] uuid={UUIDS[0]} result=failure status=ERROR", out.getvalue()) + self.assertEqual((stats.images_ok, stats.images_failed), (0, 1)) + + def test_bad_image_does_not_stop_batch(self): + sets, s3 = self.batch() + stats = main.PollerStats(None) + with self.fail_first_with(ValueError("metadata.servertime missing")), \ + contextlib.redirect_stdout(io.StringIO()): + main._process_ready_batch(self.cfg, sets, s3, stats) + self.assertEqual(self.processed, UUIDS) + self.assertEqual(self.status_calls, [(UUIDS[0], "ERROR", "metadata.servertime missing")]) + self.assertEqual((stats.images_ok, stats.images_failed), (2, 1)) + + def test_db_down_propagates_as_transient(self): + sets, s3 = self.batch() + + def no_db(cfg): + raise psycopg.OperationalError("connection timeout expired") + + with mock.patch.object(main, "pg_connect", no_db): + with self.assertRaises(psycopg.OperationalError) as ctx: + main._process_ready_batch(self.cfg, sets, s3) + self.assertTrue(main.is_transient_error(ctx.exception)) + + def test_run_once_parallel_raises_when_a_worker_hits_outage(self): + cfg = make_cfg(self.tmp.name, parallel_workers=3) + s3 = FakeS3(entrance_objects(*UUIDS)) + with self.fail_first_with(s3_unavailable()), contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(ClientError): + main.run_once(cfg, main.PollerStats(None), s3c=s3) + self.assertEqual(sorted(self.processed), UUIDS) # die anderen Worker laufen zu Ende + + +class S3ClientConfigTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cfg = make_cfg(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_poller_client_has_explicit_timeouts_and_retries(self): + conf = main.S3(self.cfg, main.s3_client_config(self.cfg)).client.meta.config + self.assertEqual(conf.connect_timeout, 10) + self.assertEqual(conf.read_timeout, 30) + self.assertEqual(conf.retries.get("mode"), "standard") + self.assertEqual(conf.retries.get("total_max_attempts"), 4) + self.assertNotIn("max_attempts", conf.retries) + self.assertEqual(conf.signature_version, "s3v4") + self.assertEqual(conf.s3.get("addressing_style"), "path") + + def test_default_client_unchanged_for_gundi_path(self): + import boto3 + from botocore.config import Config + + # exakt wie vor der Aenderung gebaut + before = boto3.client( + "s3", + endpoint_url=self.cfg.s3_endpoint, + aws_access_key_id=self.cfg.s3_access_key, + aws_secret_access_key=self.cfg.s3_secret_key, + config=Config(max_pool_connections=self.cfg.s3_max_pool_connections), + ).meta.config + now = main.S3(self.cfg).client.meta.config + for attr in ("connect_timeout", "read_timeout", "retries", "max_pool_connections", + "signature_version", "s3", "region_name"): + with self.subTest(attr=attr): + self.assertEqual(getattr(now, attr), getattr(before, attr)) + self.assertEqual(now.retries, {"mode": "legacy"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/entrance_inventory.py b/tools/entrance_inventory.py new file mode 100644 index 0000000..bc0401e --- /dev/null +++ b/tools/entrance_inventory.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend. + +Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein: + + complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar + jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json) + json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json) + cat_only nur _categories.json + foreign Name passt zu keinem ICU-Muster + +Alter = Alter des juengsten Objekts der Gruppe (konservativ). + + python -m tools.entrance_inventory # Zusammenfassung + python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job + python -m tools.entrance_inventory --list jpg_only --min-age-days 7 + python -m tools.entrance_inventory --db --csv /tmp/entrance.csv + +Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts. +""" +import argparse +import csv +import os +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Optional + +from main import CAT_RE, S3, UUID_RE, load_config + +TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"] +AGE_BUCKETS = [ + ("<1h", 3600), + ("1h-1d", 86400), + ("1-7d", 7 * 86400), + ("7-30d", 30 * 86400), + (">30d", None), +] + + +@dataclass +class Group: + uuid: Optional[str] + parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size) + + @property + def kind(self) -> str: + if self.uuid is None: + return "foreign" + has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts + if has_jpg and has_meta: + return "complete" + if has_jpg: + return "jpg_only" + if has_meta: + return "json_only" + return "cat_only" + + @property + def newest(self) -> datetime: + return max(p[1] for p in self.parts.values()) + + @property + def size(self) -> int: + return sum(p[2] for p in self.parts.values()) + + @property + def keys(self) -> list[str]: + return [p[0] for p in self.parts.values()] + + +def age_bucket(age_s: float) -> str: + for name, limit in AGE_BUCKETS: + if limit is None or age_s < limit: + return name + return AGE_BUCKETS[-1][0] + + +def collect(s3: S3) -> list[Group]: + groups: dict[str, Group] = {} + foreign: list[Group] = [] + for obj in s3.list_entrance_objects(): + key = obj["Key"] + name = key.split("/")[-1] + entry = (key, obj["LastModified"], obj.get("Size", 0)) + m = UUID_RE.match(name) + if m: + typ = "jpg" if m.group(2) == "jpg" else "meta" + groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry + continue + m = CAT_RE.match(name) + if m: + groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry + continue + foreign.append(Group(None, {"other": entry})) + return list(groups.values()) + foreign + + +def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]: + import psycopg + + out: dict[str, tuple] = {} + with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur: + cur.execute("SET TRANSACTION READ ONLY") + for i in range(0, len(uuids), 5000): + cur.execute( + """ + SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200) + FROM remote_cam.import_job + WHERE image_uuid = ANY(%s) + """, + (uuids[i:i + 5000],), + ) + for row in cur.fetchall(): + out[row[0]] = row[1:] + return out + + +def fmt_ts(dt: Optional[datetime]) -> str: + return dt.strftime("%Y-%m-%d %H:%M") if dt else "-" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)") + ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen") + ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben") + ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist") + ap.add_argument("--csv", help="alle Gruppen als CSV schreiben") + args = ap.parse_args() + + cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml")) + now = datetime.now(timezone.utc) + groups = collect(S3(cfg)) + + status: dict[str, tuple] = {} + if args.db: + status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid]) + + min_age_s = args.min_age_days * 86400 + selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s] + + # --- Zusammenfassung: Typ x Altersklasse + by_kind: dict[str, list[Group]] = defaultdict(list) + for g in selected: + by_kind[g.kind].append(g) + + bucket_names = [b[0] for b in AGE_BUCKETS] + print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC" + f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen" + + (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")") + print() + header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes" + print(header) + print("-" * len(header)) + for kind in TYPES: + gs = by_kind.get(kind, []) + if not gs: + continue + buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs) + mb = sum(g.size for g in gs) / 1e6 + print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names) + + f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}") + + with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only") + if with_cat: + print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items()))) + + if args.db: + print("\nimport_job-Status je Typ:") + for kind in TYPES[:-1]: + gs = by_kind.get(kind, []) + if gs: + c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs) + print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common())) + + # Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang. + if by_kind.get("foreign"): + print("\nFremde Objekte:") + for g in sorted(by_kind["foreign"], key=lambda x: x.newest): + key, lm, size = g.parts["other"] + print(f" {fmt_ts(lm)} {size:>10} B {key}") + + if args.list: + print(f"\n# Keys vom Typ {args.list}:") + for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest): + for key in sorted(g.keys): + print(key) + + if args.csv: + with open(args.csv, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes", + "job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"]) + for g in sorted(selected, key=lambda x: (x.kind, x.newest)): + st = status.get(g.uuid) if g.uuid else None + w.writerow([ + g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts, + g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size, + st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "", + st[3] if st else "", " ".join(sorted(g.keys)), + ]) + print(f"\nCSV geschrieben: {args.csv}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())