Make legacy poller resilient to S3/DB outages
The poller crashed on every S3 or DB error (1162 restarts during the fsn1 object storage degradation on 2026-09-21). Now: - main loop catches errors per cycle, classifies them (S3 5xx/throttling, connection/timeouts, psycopg.OperationalError = transient; everything else logged with traceback) and retries with exponential backoff (5 s .. 5 min, reset after success); db_init runs until it succeeds once - explicit botocore config for the poller: connect/read timeouts, standard retries (total attempts), s3v4, path addressing; S3(cfg) without a config keeps the previous client for the Gundi path - DB connect timeout + TCP keepalives - one S3 client per cycle shared by all workers - a batch stops after a transient S3/DB error (image stays ERROR as before) - no busy loop when ready sets keep failing - [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck - tools/entrance_inventory.py: read-only listing of icu/entrance orphans - unit tests with simulated S3/DB failures Per-image transaction, status names/order, keys and paths are unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,45 @@ docker compose up -d --build
|
||||
docker compose logs -f
|
||||
```
|
||||
|
||||
### Stoerungen und Lebenszeichen (Poller)
|
||||
|
||||
Fehler eines Zyklus (S3 5xx/Throttling, Netz, DB weg) beenden den Poller nicht:
|
||||
er loggt eine Zeile `[CYCLE_FAIL] kind=transient ...` und versucht es mit
|
||||
exponentiellem Backoff erneut (`backoff_initial_seconds` bis `backoff_max_seconds`,
|
||||
Reset nach Erfolg, dann `[RECOVERED]`). Unerwartete Fehler (`kind=unexpected`,
|
||||
z. B. AccessDenied oder Programmierfehler) werden zusaetzlich mit Traceback geloggt.
|
||||
Ein Bild, bei dem S3/DB wegbricht, bleibt wie bisher auf `ERROR`; der Rest des
|
||||
Batches wartet auf den naechsten Zyklus.
|
||||
|
||||
Alle `heartbeat_seconds` erscheint `[HEARTBEAT]` mit Zaehlern (Zyklen, Bilder,
|
||||
Fehler, letzter Erfolg). Der Docker-Healthcheck prueft, ob die Hauptschleife
|
||||
vorankommt (`/tmp/melesicumover.heartbeat` juenger als 15 min):
|
||||
|
||||
```bash
|
||||
docker inspect -f '{{.State.Health.Status}} restarts={{.RestartCount}}' melesicumover
|
||||
docker exec melesicumover cat /tmp/melesicumover.heartbeat
|
||||
```
|
||||
|
||||
S3-Timeouts/-Retries stehen unter `app.s3_*` in `config.yaml`.
|
||||
|
||||
### Eingang pruefen (verwaiste Objekte)
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps melesicumover python -m tools.entrance_inventory --db
|
||||
```
|
||||
|
||||
Listet `icu/entrance/` nach Typ (vollstaendig, JPG ohne JSON, JSON ohne JPG,
|
||||
nur Kategorien, fremd) und Alter, optional mit `import_job`-Status. Nur lesend.
|
||||
|
||||
### Tests
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python -m unittest discover -s tests -v # oder: python -m pytest tests
|
||||
```
|
||||
|
||||
Simulierte S3-/DB-Stoerungen fuer Backoff und Fehlerbehandlung; weder Netz noch DB noetig.
|
||||
|
||||
## Gundi-Push-Pfad (`gundi-receiver`)
|
||||
|
||||
ICU pusht Events ueber eine Gundi-v2-kompatible API. Der Service `gundi-receiver`
|
||||
|
||||
@@ -39,6 +39,15 @@ app:
|
||||
enable_deermapper_api: false
|
||||
deermapper_api_timeout_seconds: 20
|
||||
deermapper_api_image_field: "image"
|
||||
# Robustheit (nur Poller): S3-Timeouts/Retries, Backoff bei Stoerungen, Lebenszeichen
|
||||
s3_connect_timeout_seconds: 10
|
||||
s3_read_timeout_seconds: 30
|
||||
s3_retry_mode: "standard" # standard | adaptive | legacy
|
||||
s3_max_attempts: 4 # inkl. erstem Versuch
|
||||
pg_connect_timeout_seconds: 10
|
||||
backoff_initial_seconds: 5 # 5 s, 10 s, 20 s ... nach fehlgeschlagenem Zyklus
|
||||
backoff_max_seconds: 300 # hoechstens 5 min; Reset nach erfolgreichem Zyklus
|
||||
heartbeat_seconds: 300 # [HEARTBEAT]-Logzeile mit Zaehlern
|
||||
|
||||
# Gundi-v2-Push-Empfaenger (Service gundi-receiver).
|
||||
# api key -> via ENV (.env): GUNDI_API_KEY (min. 24 Zeichen)
|
||||
|
||||
@@ -10,6 +10,14 @@ services:
|
||||
environment:
|
||||
CONFIG_YAML: /app/config.yaml
|
||||
DEBUG: "0"
|
||||
# Lebenszeichen: die Hauptschleife schreibt die Heartbeat-Datei bei jedem Zyklus
|
||||
# und jedem Bild. Laenger als 15 min unveraendert = Schleife haengt (Backoff max. 5 min).
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/melesicumover.heartbeat') < 900 else 1)"]
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 120s
|
||||
# Kein Bind-Mount des Quellcodes mehr (der Code steckt im Image) und
|
||||
# standardmaessig KEIN offener Debug-Port.
|
||||
#
|
||||
|
||||
@@ -6,6 +6,8 @@ import json
|
||||
import time
|
||||
import tempfile
|
||||
import subprocess
|
||||
import threading
|
||||
import traceback
|
||||
import uuid as uuidlib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
@@ -17,6 +19,12 @@ from urllib import request as urlrequest
|
||||
import boto3
|
||||
import psycopg
|
||||
from botocore.config import Config as BotoClientConfig
|
||||
from botocore.exceptions import (
|
||||
ClientError,
|
||||
ConnectionError as BotoConnectionError,
|
||||
HTTPClientError,
|
||||
IncompleteReadError,
|
||||
)
|
||||
from dateutil import tz
|
||||
from PIL import Image, ImageOps, ImageEnhance
|
||||
import pytesseract
|
||||
@@ -73,6 +81,17 @@ class AppConfig:
|
||||
deermapper_api_timeout_seconds: int = 20
|
||||
deermapper_api_image_field: str = "image"
|
||||
|
||||
# Robustheit Poller: S3-Timeouts/Retries, DB-Connect-Timeout, Backoff, Lebenszeichen
|
||||
s3_connect_timeout_seconds: float = 10.0
|
||||
s3_read_timeout_seconds: float = 30.0
|
||||
s3_retry_mode: str = "standard"
|
||||
s3_max_attempts: int = 4
|
||||
pg_connect_timeout_seconds: int = 10
|
||||
backoff_initial_seconds: float = 5.0
|
||||
backoff_max_seconds: float = 300.0
|
||||
heartbeat_seconds: int = 300
|
||||
heartbeat_file: str = "/tmp/melesicumover.heartbeat"
|
||||
|
||||
|
||||
def load_config(path: str) -> AppConfig:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
@@ -128,6 +147,15 @@ def load_config(path: str) -> AppConfig:
|
||||
deermapper_api_key=env_or("DEERMAPPER_API_KEY", deermapper.get("apiKey", "")),
|
||||
deermapper_api_timeout_seconds=int(app.get("deermapper_api_timeout_seconds", 20)),
|
||||
deermapper_api_image_field=app.get("deermapper_api_image_field", "image"),
|
||||
s3_connect_timeout_seconds=float(app.get("s3_connect_timeout_seconds", 10)),
|
||||
s3_read_timeout_seconds=float(app.get("s3_read_timeout_seconds", 30)),
|
||||
s3_retry_mode=str(app.get("s3_retry_mode", "standard")),
|
||||
s3_max_attempts=max(1, int(app.get("s3_max_attempts", 4))),
|
||||
pg_connect_timeout_seconds=max(1, int(app.get("pg_connect_timeout_seconds", 10))),
|
||||
backoff_initial_seconds=max(0.1, float(app.get("backoff_initial_seconds", 5))),
|
||||
backoff_max_seconds=max(1.0, float(app.get("backoff_max_seconds", 300))),
|
||||
heartbeat_seconds=max(10, int(app.get("heartbeat_seconds", 300))),
|
||||
heartbeat_file=app.get("heartbeat_file", "/tmp/melesicumover.heartbeat"),
|
||||
)
|
||||
|
||||
# Fail-fast: Secrets kommen ausschliesslich aus der Umgebung (.env). Fehlen sie,
|
||||
@@ -454,6 +482,26 @@ def db_init(cur):
|
||||
""")
|
||||
|
||||
|
||||
def pg_connect(cfg: AppConfig) -> psycopg.Connection:
|
||||
# connect_timeout + TCP-Keepalives: ein weggebrochener DB-Host blockiert den
|
||||
# Poller nicht unbegrenzt, sondern endet als OperationalError (-> Backoff).
|
||||
return psycopg.connect(
|
||||
cfg.pg_dsn,
|
||||
connect_timeout=cfg.pg_connect_timeout_seconds,
|
||||
keepalives=1,
|
||||
keepalives_idle=60,
|
||||
keepalives_interval=10,
|
||||
keepalives_count=3,
|
||||
)
|
||||
|
||||
|
||||
def init_db_schema(cfg: AppConfig):
|
||||
with pg_connect(cfg) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
|
||||
def job_upsert_new(cur, uuid: str, jpg_key: str, meta_key: str, cat_key: Optional[str]):
|
||||
cur.execute("""
|
||||
INSERT INTO remote_cam.import_job (image_uuid, status, entrance_jpg_key, entrance_meta_key, entrance_cat_key, has_categories)
|
||||
@@ -676,15 +724,34 @@ def insert_api_response(cur, uuid: str, success: bool, message: str):
|
||||
# -----------------------
|
||||
# S3 ops wrapper
|
||||
# -----------------------
|
||||
def s3_client_config(cfg: AppConfig) -> BotoClientConfig:
|
||||
"""Explizite Timeouts/Retries fuer den Poller (botocore-Default: legacy, 60 s Read-Timeout).
|
||||
|
||||
Pfad-Adressierung + s3v4 wie im Portal; die Adressierung entspricht dem bisherigen
|
||||
Verhalten mit eigenem endpoint_url.
|
||||
"""
|
||||
return BotoClientConfig(
|
||||
max_pool_connections=cfg.s3_max_pool_connections,
|
||||
connect_timeout=cfg.s3_connect_timeout_seconds,
|
||||
read_timeout=cfg.s3_read_timeout_seconds,
|
||||
# total_max_attempts = Versuche inkl. erstem (botocores "max_attempts" zaehlt nur Wiederholungen)
|
||||
retries={"mode": cfg.s3_retry_mode, "total_max_attempts": cfg.s3_max_attempts},
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
)
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self, cfg: AppConfig):
|
||||
def __init__(self, cfg: AppConfig, client_config: Optional[BotoClientConfig] = None):
|
||||
self.cfg = cfg
|
||||
# Ohne client_config bleibt die bisherige Client-Konfiguration (so nutzt sie der
|
||||
# Gundi-Pfad); der Poller uebergibt s3_client_config(cfg).
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=cfg.s3_endpoint,
|
||||
aws_access_key_id=cfg.s3_access_key,
|
||||
aws_secret_access_key=cfg.s3_secret_key,
|
||||
config=BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
|
||||
config=client_config or BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
|
||||
)
|
||||
|
||||
def list_entrance_objects(self):
|
||||
@@ -933,15 +1000,15 @@ def cleanup_entrance_set(s3: S3, s: EntranceSet) -> int:
|
||||
return s3.delete_keys(to_delete)
|
||||
|
||||
|
||||
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
s3c = S3(cfg)
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet], s3c: S3, stats: Optional["PollerStats"] = None):
|
||||
with pg_connect(cfg) as db:
|
||||
for s in batch:
|
||||
start = time.perf_counter()
|
||||
result = "failure"
|
||||
final_status = "UNKNOWN"
|
||||
err_text = None
|
||||
cleanup_deleted = None
|
||||
abort = None
|
||||
try:
|
||||
skip_duplicate = False
|
||||
with db.transaction():
|
||||
@@ -963,6 +1030,11 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
job_set_status(cur, s.uuid, "ERROR", err=str(e))
|
||||
final_status = "ERROR"
|
||||
err_text = str(e)
|
||||
# S3/DB gestoert: Bild bleibt wie bisher auf ERROR, der Rest des Batches wird
|
||||
# nicht mehr angefasst (naechster Zyklus nach Backoff) statt jedes Bild
|
||||
# einzeln gegen die Stoerung laufen zu lassen.
|
||||
if is_transient_error(e):
|
||||
abort = e
|
||||
elapsed_ms = int((time.perf_counter() - start) * 1000)
|
||||
if err_text:
|
||||
print(
|
||||
@@ -976,19 +1048,25 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
)
|
||||
else:
|
||||
print(f"[RESULT] uuid={s.uuid} result={result} status={final_status} process_ms={elapsed_ms}")
|
||||
if stats is not None:
|
||||
stats.image_done(result == "success")
|
||||
if abort is not None:
|
||||
raise abort
|
||||
if cfg.job_sleep_seconds > 0:
|
||||
time.sleep(cfg.job_sleep_seconds)
|
||||
|
||||
|
||||
def run_once(cfg: AppConfig) -> int:
|
||||
s3c = S3(cfg)
|
||||
def run_once(cfg: AppConfig, stats: Optional["PollerStats"] = None, s3c: Optional[S3] = None) -> int:
|
||||
# Ein Client pro Zyklus fuer alle Worker: boto3-Clients sind threadsicher, das
|
||||
# parallele Anlegen ueber die Default-Session dagegen nicht.
|
||||
s3c = s3c or S3(cfg, s3_client_config(cfg))
|
||||
ready = collect_ready_sets(s3c, cfg)
|
||||
if not ready:
|
||||
return 0
|
||||
|
||||
workers = max(1, min(cfg.parallel_workers, len(ready)))
|
||||
if workers == 1:
|
||||
_process_ready_batch(cfg, ready)
|
||||
_process_ready_batch(cfg, ready, s3c, stats)
|
||||
return len(ready)
|
||||
|
||||
batches: list[list[EntranceSet]] = [[] for _ in range(workers)]
|
||||
@@ -996,13 +1074,237 @@ def run_once(cfg: AppConfig) -> int:
|
||||
batches[i % workers].append(s)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futures = [pool.submit(_process_ready_batch, cfg, batch) for batch in batches if batch]
|
||||
futures = [pool.submit(_process_ready_batch, cfg, batch, s3c, stats) for batch in batches if batch]
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
|
||||
return len(ready)
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Robustheit: Fehlerklassen, Backoff, Lebenszeichen
|
||||
# -----------------------
|
||||
TRANSIENT_S3_CODES = {
|
||||
"ServiceUnavailable", "SlowDown", "Throttling", "ThrottlingException", "TooManyRequests",
|
||||
"RequestLimitExceeded", "RequestTimeout", "InternalError", "500", "502", "503", "504",
|
||||
}
|
||||
|
||||
|
||||
def is_transient_error(exc: Optional[BaseException]) -> bool:
|
||||
"""True fuer Stoerungen, die ohne Codeaenderung vorbeigehen: S3 5xx/Throttling,
|
||||
Verbindungs-/Timeout-Fehler, DB weg (psycopg.OperationalError). Alles andere
|
||||
(4xx wie AccessDenied, Programmierfehler, SQL-Fehler) gilt als unerwartet.
|
||||
Folgt der expliziten Ursache (raise ... from ...)."""
|
||||
seen = set()
|
||||
while exc is not None and id(exc) not in seen:
|
||||
seen.add(id(exc))
|
||||
if isinstance(exc, ClientError):
|
||||
err = exc.response.get("Error") or {}
|
||||
http = (exc.response.get("ResponseMetadata") or {}).get("HTTPStatusCode") or 0
|
||||
if str(err.get("Code", "")) in TRANSIENT_S3_CODES or http >= 500 or http == 429:
|
||||
return True
|
||||
elif isinstance(exc, (BotoConnectionError, HTTPClientError, IncompleteReadError)):
|
||||
return True
|
||||
elif isinstance(exc, psycopg.OperationalError):
|
||||
return True
|
||||
elif isinstance(exc, (ConnectionError, TimeoutError)):
|
||||
return True
|
||||
exc = exc.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def describe_error(exc: BaseException, limit: int = 300) -> str:
|
||||
text = " ".join(str(exc).split())
|
||||
if len(text) > limit:
|
||||
text = text[:limit] + "..."
|
||||
return f"{type(exc).__name__}: {text}"
|
||||
|
||||
|
||||
class Backoff:
|
||||
"""Exponentiell: initial, 2x, 4x ... bis maximum; reset() nach einem erfolgreichen Zyklus."""
|
||||
|
||||
def __init__(self, initial: float, maximum: float, factor: float = 2.0):
|
||||
self.initial = initial
|
||||
self.maximum = maximum
|
||||
self.factor = factor
|
||||
self.failures = 0
|
||||
|
||||
def next_delay(self) -> float:
|
||||
# Exponent begrenzen: bei tagelanger Stoerung sonst OverflowError.
|
||||
delay = min(self.maximum, self.initial * (self.factor ** min(self.failures, 32)))
|
||||
self.failures += 1
|
||||
return delay
|
||||
|
||||
def reset(self):
|
||||
self.failures = 0
|
||||
|
||||
|
||||
def _utc_iso(ts: Optional[float]) -> str:
|
||||
if ts is None:
|
||||
return "never"
|
||||
return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
class PollerStats:
|
||||
"""Zaehler fuer Lebenszeichen-Log und Healthcheck. Threadsicher, weil die Worker je Bild melden.
|
||||
|
||||
Die Heartbeat-Datei wird bei jedem Zyklus und jedem Bild neu geschrieben; ihr mtime
|
||||
zeigt dem Docker-Healthcheck, dass die Schleife vorankommt (wartet vs. haengt)."""
|
||||
|
||||
def __init__(self, heartbeat_file: Optional[str], clock=time.time):
|
||||
self._lock = threading.Lock()
|
||||
self._clock = clock
|
||||
self.heartbeat_file = heartbeat_file
|
||||
self.started = clock()
|
||||
self.cycles = 0
|
||||
self.cycles_failed = 0
|
||||
self.images_ok = 0
|
||||
self.images_failed = 0
|
||||
self.errors_transient = 0
|
||||
self.errors_other = 0
|
||||
self.consecutive_failures = 0
|
||||
self.last_success: Optional[float] = None
|
||||
self.last_error: Optional[str] = None
|
||||
|
||||
def image_done(self, ok: bool):
|
||||
with self._lock:
|
||||
if ok:
|
||||
self.images_ok += 1
|
||||
else:
|
||||
self.images_failed += 1
|
||||
self._write_heartbeat()
|
||||
|
||||
def cycle_ok(self):
|
||||
with self._lock:
|
||||
self.cycles += 1
|
||||
self.consecutive_failures = 0
|
||||
self.last_success = self._clock()
|
||||
self._write_heartbeat()
|
||||
|
||||
def cycle_failed(self, transient: bool, message: str):
|
||||
with self._lock:
|
||||
self.cycles += 1
|
||||
self.cycles_failed += 1
|
||||
self.consecutive_failures += 1
|
||||
if transient:
|
||||
self.errors_transient += 1
|
||||
else:
|
||||
self.errors_other += 1
|
||||
self.last_error = message
|
||||
self._write_heartbeat()
|
||||
|
||||
def touch(self):
|
||||
with self._lock:
|
||||
self._write_heartbeat()
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
now = self._clock()
|
||||
return {
|
||||
"ts": _utc_iso(now),
|
||||
"uptime_s": int(now - self.started),
|
||||
"cycles": self.cycles,
|
||||
"cycles_failed": self.cycles_failed,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"images_ok": self.images_ok,
|
||||
"images_failed": self.images_failed,
|
||||
"errors_transient": self.errors_transient,
|
||||
"errors_other": self.errors_other,
|
||||
"last_success": _utc_iso(self.last_success),
|
||||
"last_success_age_s": None if self.last_success is None else int(now - self.last_success),
|
||||
"last_error": self.last_error,
|
||||
}
|
||||
|
||||
def summary(self) -> str:
|
||||
s = self.snapshot()
|
||||
age = "-" if s["last_success_age_s"] is None else f"{s['last_success_age_s']}s"
|
||||
return (
|
||||
f"uptime={s['uptime_s']}s cycles={s['cycles']} cycles_failed={s['cycles_failed']} "
|
||||
f"consecutive_failures={s['consecutive_failures']} images_ok={s['images_ok']} "
|
||||
f"images_failed={s['images_failed']} errors_transient={s['errors_transient']} "
|
||||
f"errors_other={s['errors_other']} last_success={s['last_success']} last_success_age={age}"
|
||||
)
|
||||
|
||||
def _write_heartbeat(self):
|
||||
if not self.heartbeat_file:
|
||||
return
|
||||
try:
|
||||
tmp = f"{self.heartbeat_file}.tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(self.snapshot(), f)
|
||||
os.replace(tmp, self.heartbeat_file)
|
||||
except OSError as e:
|
||||
print(f"[WARN] heartbeat file not writable: {e}")
|
||||
|
||||
|
||||
def poll_forever(
|
||||
cfg: AppConfig,
|
||||
*,
|
||||
cycle=run_once,
|
||||
init_schema=init_db_schema,
|
||||
sleep=time.sleep,
|
||||
clock=time.time,
|
||||
stats: Optional[PollerStats] = None,
|
||||
max_iterations: Optional[int] = None,
|
||||
) -> PollerStats:
|
||||
"""Hauptschleife. Faengt Fehler je Zyklus ab und wartet mit Backoff - S3- oder DB-Ausfaelle
|
||||
beenden den Prozess nicht mehr. db_init laeuft einmal erfolgreich, danach nicht erneut.
|
||||
Transaktion pro Bild und Statusfolge liegen unveraendert in _process_ready_batch/process_one.
|
||||
"""
|
||||
stats = stats or PollerStats(cfg.heartbeat_file, clock=clock)
|
||||
backoff = Backoff(cfg.backoff_initial_seconds, cfg.backoff_max_seconds)
|
||||
schema_ready = False
|
||||
failing_since: Optional[float] = None
|
||||
last_heartbeat_log = clock()
|
||||
iterations = 0
|
||||
|
||||
while max_iterations is None or iterations < max_iterations:
|
||||
iterations += 1
|
||||
stats.touch()
|
||||
ok_before = stats.images_ok
|
||||
try:
|
||||
if not schema_ready:
|
||||
init_schema(cfg)
|
||||
schema_ready = True
|
||||
n = cycle(cfg, stats)
|
||||
except Exception as e:
|
||||
transient = is_transient_error(e)
|
||||
msg = describe_error(e)
|
||||
delay = backoff.next_delay()
|
||||
stats.cycle_failed(transient, msg)
|
||||
if failing_since is None:
|
||||
failing_since = clock()
|
||||
kind = "transient" if transient else "unexpected"
|
||||
print(
|
||||
f"[CYCLE_FAIL] kind={kind} consecutive={backoff.failures} "
|
||||
f"retry_in={delay:.0f}s error={msg}"
|
||||
)
|
||||
if not transient:
|
||||
traceback.print_exc()
|
||||
n = None
|
||||
else:
|
||||
if failing_since is not None:
|
||||
print(
|
||||
f"[RECOVERED] after_failures={backoff.failures} "
|
||||
f"outage_s={int(clock() - failing_since)}"
|
||||
)
|
||||
failing_since = None
|
||||
backoff.reset()
|
||||
stats.cycle_ok()
|
||||
delay = None
|
||||
|
||||
if clock() - last_heartbeat_log >= cfg.heartbeat_seconds:
|
||||
print(f"[HEARTBEAT] {stats.summary()}")
|
||||
last_heartbeat_log = clock()
|
||||
|
||||
if delay is not None:
|
||||
sleep(delay)
|
||||
elif n == 0 or stats.images_ok == ok_before:
|
||||
# Nichts zu tun - oder nur Saetze, die erneut scheitern: nicht im Leerlauf kreiseln.
|
||||
sleep(cfg.poll_seconds)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
if os.environ.get("DEBUG") == "1":
|
||||
import debugpy
|
||||
@@ -1013,15 +1315,13 @@ def main():
|
||||
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
|
||||
cfg = load_config(cfg_path)
|
||||
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
while True:
|
||||
n = run_once(cfg)
|
||||
if n == 0:
|
||||
time.sleep(cfg.poll_seconds)
|
||||
print(
|
||||
f"[START] poll={cfg.poll_seconds}s workers={cfg.parallel_workers} "
|
||||
f"s3_timeout=connect:{cfg.s3_connect_timeout_seconds:g}s/read:{cfg.s3_read_timeout_seconds:g}s "
|
||||
f"s3_retries={cfg.s3_retry_mode}/{cfg.s3_max_attempts} pg_connect_timeout={cfg.pg_connect_timeout_seconds}s "
|
||||
f"backoff={cfg.backoff_initial_seconds:g}-{cfg.backoff_max_seconds:g}s heartbeat={cfg.heartbeat_seconds}s"
|
||||
)
|
||||
poll_forever(cfg)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,440 @@
|
||||
"""Backoff und Fehlerbehandlung des Legacy-Pollers mit simulierten S3-/DB-Stoerungen.
|
||||
|
||||
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
||||
|
||||
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import psycopg # noqa: E402
|
||||
from botocore.exceptions import ( # noqa: E402
|
||||
ClientError,
|
||||
ConnectTimeoutError,
|
||||
ConnectionClosedError,
|
||||
EndpointConnectionError,
|
||||
ReadTimeoutError,
|
||||
)
|
||||
|
||||
import main # noqa: E402
|
||||
|
||||
|
||||
def client_error(code: str, http: int, op: str = "ListObjectsV2") -> ClientError:
|
||||
return ClientError({"Error": {"Code": code, "Message": code}, "ResponseMetadata": {"HTTPStatusCode": http}}, op)
|
||||
|
||||
|
||||
def s3_unavailable() -> ClientError:
|
||||
return client_error("ServiceUnavailable", 503)
|
||||
|
||||
|
||||
def make_cfg(tmpdir: str, **overrides) -> main.AppConfig:
|
||||
params = dict(
|
||||
s3_endpoint="https://s3.invalid/",
|
||||
s3_access_key="test",
|
||||
s3_secret_key="test",
|
||||
s3_bucket="bucket",
|
||||
pg_dsn="postgresql://invalid/db",
|
||||
poll_seconds=30,
|
||||
parallel_workers=1,
|
||||
backoff_initial_seconds=5,
|
||||
backoff_max_seconds=300,
|
||||
heartbeat_seconds=300,
|
||||
heartbeat_file=os.path.join(tmpdir, "heartbeat.json"),
|
||||
)
|
||||
params.update(overrides)
|
||||
return main.AppConfig(**params)
|
||||
|
||||
|
||||
class FakeClock:
|
||||
"""Zeit laeuft nur durch sleep() - Tests laufen ohne echtes Warten."""
|
||||
|
||||
def __init__(self, start: float = 1_000_000.0):
|
||||
self.now = start
|
||||
self.sleeps: list[float] = []
|
||||
|
||||
def time(self) -> float:
|
||||
return self.now
|
||||
|
||||
def sleep(self, seconds: float):
|
||||
self.sleeps.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
class ScriptedCycle:
|
||||
"""Zyklus-Ersatz: arbeitet eine Liste ab (Exception -> werfen, int -> zurueckgeben)."""
|
||||
|
||||
def __init__(self, steps):
|
||||
self.steps = list(steps)
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, cfg, stats):
|
||||
self.calls += 1
|
||||
step = self.steps.pop(0) if self.steps else 0
|
||||
if isinstance(step, BaseException):
|
||||
raise step
|
||||
if callable(step):
|
||||
return step(stats)
|
||||
return step
|
||||
|
||||
|
||||
class FakeS3:
|
||||
"""Simulierter S3-Client: Listing liefert Objekte oder wirft den angegebenen Fehler."""
|
||||
|
||||
def __init__(self, objects=None, list_error: BaseException = None):
|
||||
self.objects = objects or []
|
||||
self.list_error = list_error
|
||||
|
||||
def list_entrance_objects(self):
|
||||
if self.list_error is not None:
|
||||
raise self.list_error
|
||||
yield from self.objects
|
||||
|
||||
|
||||
class FakeConn:
|
||||
"""Simulierte psycopg-Verbindung; die SQL-Funktionen werden in den Tests gepatcht."""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
@contextlib.contextmanager
|
||||
def transaction(self):
|
||||
yield
|
||||
|
||||
def cursor(self):
|
||||
return contextlib.nullcontext(object())
|
||||
|
||||
|
||||
def entrance_objects(*uuids):
|
||||
old = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
objs = []
|
||||
for u in uuids:
|
||||
objs.append({"Key": f"icu/entrance/{u}.jpg", "LastModified": old})
|
||||
objs.append({"Key": f"icu/entrance/{u}.json", "LastModified": old})
|
||||
return objs
|
||||
|
||||
|
||||
UUIDS = [f"00000000-0000-4000-8000-00000000000{i}" for i in range(1, 4)]
|
||||
|
||||
|
||||
class BackoffTest(unittest.TestCase):
|
||||
def test_doubles_up_to_maximum(self):
|
||||
b = main.Backoff(5, 300)
|
||||
self.assertEqual([b.next_delay() for _ in range(9)], [5, 10, 20, 40, 80, 160, 300, 300, 300])
|
||||
|
||||
def test_reset_starts_again_at_initial(self):
|
||||
b = main.Backoff(5, 300)
|
||||
for _ in range(4):
|
||||
b.next_delay()
|
||||
b.reset()
|
||||
self.assertEqual(b.failures, 0)
|
||||
self.assertEqual(b.next_delay(), 5)
|
||||
|
||||
def test_long_outage_does_not_overflow(self):
|
||||
b = main.Backoff(5, 300)
|
||||
for _ in range(5000):
|
||||
delay = b.next_delay()
|
||||
self.assertEqual(delay, 300)
|
||||
|
||||
|
||||
class ErrorClassificationTest(unittest.TestCase):
|
||||
def test_transient(self):
|
||||
cases = [
|
||||
s3_unavailable(),
|
||||
client_error("SlowDown", 503),
|
||||
client_error("InternalError", 500),
|
||||
client_error("TooManyRequests", 429),
|
||||
client_error("Whatever", 502),
|
||||
EndpointConnectionError(endpoint_url="https://s3.invalid"),
|
||||
ConnectTimeoutError(endpoint_url="https://s3.invalid"),
|
||||
ReadTimeoutError(endpoint_url="https://s3.invalid"),
|
||||
ConnectionClosedError(endpoint_url="https://s3.invalid"),
|
||||
psycopg.OperationalError("connection refused"),
|
||||
psycopg.errors.AdminShutdown("terminating connection due to administrator command"),
|
||||
ConnectionResetError(),
|
||||
TimeoutError(),
|
||||
]
|
||||
for exc in cases:
|
||||
with self.subTest(exc=repr(exc)):
|
||||
self.assertTrue(main.is_transient_error(exc))
|
||||
|
||||
def test_unexpected(self):
|
||||
cases = [
|
||||
client_error("AccessDenied", 403),
|
||||
client_error("NoSuchKey", 404, "GetObject"),
|
||||
client_error("NoSuchBucket", 404),
|
||||
psycopg.ProgrammingError("syntax error"),
|
||||
psycopg.errors.InsufficientPrivilege("permission denied"),
|
||||
KeyError("status"),
|
||||
ValueError("metadata.servertime missing"),
|
||||
]
|
||||
for exc in cases:
|
||||
with self.subTest(exc=repr(exc)):
|
||||
self.assertFalse(main.is_transient_error(exc))
|
||||
|
||||
def test_follows_explicit_cause_only(self):
|
||||
try:
|
||||
try:
|
||||
raise s3_unavailable()
|
||||
except ClientError as e:
|
||||
raise RuntimeError("wrapped") from e
|
||||
except RuntimeError as wrapped:
|
||||
self.assertTrue(main.is_transient_error(wrapped))
|
||||
|
||||
# Ein Bug im Fehlerpfad (nur __context__) bleibt ein unerwarteter Fehler.
|
||||
try:
|
||||
try:
|
||||
raise s3_unavailable()
|
||||
except ClientError:
|
||||
raise KeyError("bug in handler")
|
||||
except KeyError as bug:
|
||||
self.assertFalse(main.is_transient_error(bug))
|
||||
|
||||
|
||||
class PollForeverTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
self.clock = FakeClock()
|
||||
self.out = io.StringIO()
|
||||
self.err = io.StringIO()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def run_loop(self, cycle, iterations, init_schema=lambda cfg: None):
|
||||
with contextlib.redirect_stdout(self.out), contextlib.redirect_stderr(self.err):
|
||||
return main.poll_forever(
|
||||
self.cfg,
|
||||
cycle=cycle,
|
||||
init_schema=init_schema,
|
||||
sleep=self.clock.sleep,
|
||||
clock=self.clock.time,
|
||||
max_iterations=iterations,
|
||||
)
|
||||
|
||||
def test_s3_outage_backs_off_and_recovers(self):
|
||||
cycle = ScriptedCycle([s3_unavailable(), s3_unavailable(), s3_unavailable(), 0, 0, s3_unavailable(), 0])
|
||||
stats = self.run_loop(cycle, 7)
|
||||
self.assertEqual(self.clock.sleeps, [5, 10, 20, 30, 30, 5, 30])
|
||||
self.assertEqual(stats.cycles, 7)
|
||||
self.assertEqual(stats.cycles_failed, 4)
|
||||
self.assertEqual(stats.errors_transient, 4)
|
||||
self.assertEqual(stats.errors_other, 0)
|
||||
self.assertEqual(stats.consecutive_failures, 0)
|
||||
self.assertIsNotNone(stats.last_success)
|
||||
log = self.out.getvalue()
|
||||
self.assertIn("[CYCLE_FAIL] kind=transient consecutive=1 retry_in=5s error=ClientError", log)
|
||||
self.assertIn("[RECOVERED] after_failures=3 outage_s=35", log)
|
||||
self.assertNotIn("Traceback", self.err.getvalue())
|
||||
|
||||
def test_backoff_is_capped_during_long_outage(self):
|
||||
stats = self.run_loop(ScriptedCycle([s3_unavailable()] * 12), 12)
|
||||
self.assertEqual(self.clock.sleeps[-4:], [300, 300, 300, 300])
|
||||
self.assertEqual(stats.consecutive_failures, 12)
|
||||
|
||||
def test_db_unavailable_at_start_is_retried_then_schema_init_runs_once(self):
|
||||
calls = []
|
||||
|
||||
def init_schema(cfg):
|
||||
calls.append(1)
|
||||
if len(calls) <= 2:
|
||||
raise psycopg.OperationalError("could not connect to server")
|
||||
|
||||
cycle = ScriptedCycle([0, 0, 0])
|
||||
stats = self.run_loop(cycle, 5, init_schema=init_schema)
|
||||
self.assertEqual(len(calls), 3)
|
||||
self.assertEqual(cycle.calls, 3)
|
||||
self.assertEqual(self.clock.sleeps, [5, 10, 30, 30, 30])
|
||||
self.assertEqual(stats.errors_transient, 2)
|
||||
|
||||
def test_unexpected_error_is_logged_with_traceback_but_does_not_crash(self):
|
||||
cycle = ScriptedCycle([KeyError("boom"), KeyError("boom"), 0])
|
||||
stats = self.run_loop(cycle, 3)
|
||||
self.assertEqual(stats.errors_other, 2)
|
||||
self.assertEqual(self.clock.sleeps, [5, 10, 30])
|
||||
self.assertIn("[CYCLE_FAIL] kind=unexpected", self.out.getvalue())
|
||||
self.assertIn("Traceback", self.err.getvalue())
|
||||
self.assertIn("KeyError: 'boom'", self.err.getvalue())
|
||||
|
||||
def test_no_busy_loop_when_ready_sets_keep_failing(self):
|
||||
def progress(stats):
|
||||
stats.image_done(True)
|
||||
return 1
|
||||
|
||||
cycle = ScriptedCycle([2, progress, 0])
|
||||
self.run_loop(cycle, 3)
|
||||
# 2 Saetze ohne Erfolg -> Poll-Pause; Fortschritt -> sofort weiter; leer -> Poll-Pause
|
||||
self.assertEqual(self.clock.sleeps, [30, 30])
|
||||
|
||||
def test_heartbeat_log_and_file(self):
|
||||
stats = self.run_loop(ScriptedCycle([0] * 12), 12)
|
||||
beats = [line for line in self.out.getvalue().splitlines() if line.startswith("[HEARTBEAT]")]
|
||||
self.assertEqual(len(beats), 1) # Zyklus 11 laeuft bei +300 s -> genau eine Zeile
|
||||
self.assertIn("cycles=11 cycles_failed=0", beats[0])
|
||||
self.assertIn("last_success_age=0s", beats[0])
|
||||
with open(self.cfg.heartbeat_file, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
self.assertEqual(data["cycles"], stats.cycles)
|
||||
self.assertEqual(data["errors_transient"], 0)
|
||||
|
||||
|
||||
class RunOnceWithFakeS3Test(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_listing_failure_propagates_to_loop_which_survives(self):
|
||||
s3 = FakeS3(list_error=s3_unavailable())
|
||||
with self.assertRaises(ClientError):
|
||||
main.run_once(self.cfg, None, s3c=s3)
|
||||
|
||||
clock = FakeClock()
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
stats = main.poll_forever(
|
||||
self.cfg,
|
||||
cycle=lambda cfg, st: main.run_once(cfg, st, s3c=s3),
|
||||
init_schema=lambda cfg: None,
|
||||
sleep=clock.sleep,
|
||||
clock=clock.time,
|
||||
max_iterations=3,
|
||||
)
|
||||
self.assertEqual(clock.sleeps, [5, 10, 20])
|
||||
self.assertEqual(stats.errors_transient, 3)
|
||||
|
||||
def test_empty_entrance_returns_zero(self):
|
||||
self.assertEqual(main.run_once(self.cfg, None, s3c=FakeS3()), 0)
|
||||
|
||||
|
||||
class ProcessBatchTest(unittest.TestCase):
|
||||
"""Transaktion pro Bild bleibt; bei S3-/DB-Stoerung wird der Rest des Batches nicht angefasst."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
self.status_calls = []
|
||||
self.processed = []
|
||||
patches = [
|
||||
mock.patch.object(main, "pg_connect", lambda cfg: FakeConn()),
|
||||
mock.patch.object(main, "is_completed_job_materialized", lambda cur, s3, cfg, uuid: False),
|
||||
mock.patch.object(main, "job_upsert_new", lambda cur, uuid, *a: None),
|
||||
mock.patch.object(main, "job_get", lambda cur, uuid: {"status": "CLEANED"}),
|
||||
mock.patch.object(
|
||||
main, "job_set_status",
|
||||
lambda cur, uuid, status, **kw: self.status_calls.append((uuid, status, kw.get("err"))),
|
||||
),
|
||||
]
|
||||
for p in patches:
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def batch(self):
|
||||
s3 = FakeS3(entrance_objects(*UUIDS))
|
||||
return main.collect_ready_sets(s3, self.cfg), s3
|
||||
|
||||
def fail_first_with(self, exc):
|
||||
def process_one(cur, s3, cfg, s):
|
||||
self.processed.append(s.uuid)
|
||||
if s.uuid == UUIDS[0]:
|
||||
raise exc
|
||||
return mock.patch.object(main, "process_one", process_one)
|
||||
|
||||
def test_transient_s3_error_marks_image_error_and_aborts_batch(self):
|
||||
sets, s3 = self.batch()
|
||||
stats = main.PollerStats(None)
|
||||
with self.fail_first_with(client_error("ServiceUnavailable", 503, "GetObject")), \
|
||||
contextlib.redirect_stdout(io.StringIO()) as out:
|
||||
with self.assertRaises(ClientError):
|
||||
main._process_ready_batch(self.cfg, sets, s3, stats)
|
||||
self.assertEqual(self.processed, [UUIDS[0]])
|
||||
self.assertEqual([(u, st) for u, st, _ in self.status_calls], [(UUIDS[0], "ERROR")])
|
||||
self.assertIn(f"[RESULT] uuid={UUIDS[0]} result=failure status=ERROR", out.getvalue())
|
||||
self.assertEqual((stats.images_ok, stats.images_failed), (0, 1))
|
||||
|
||||
def test_bad_image_does_not_stop_batch(self):
|
||||
sets, s3 = self.batch()
|
||||
stats = main.PollerStats(None)
|
||||
with self.fail_first_with(ValueError("metadata.servertime missing")), \
|
||||
contextlib.redirect_stdout(io.StringIO()):
|
||||
main._process_ready_batch(self.cfg, sets, s3, stats)
|
||||
self.assertEqual(self.processed, UUIDS)
|
||||
self.assertEqual(self.status_calls, [(UUIDS[0], "ERROR", "metadata.servertime missing")])
|
||||
self.assertEqual((stats.images_ok, stats.images_failed), (2, 1))
|
||||
|
||||
def test_db_down_propagates_as_transient(self):
|
||||
sets, s3 = self.batch()
|
||||
|
||||
def no_db(cfg):
|
||||
raise psycopg.OperationalError("connection timeout expired")
|
||||
|
||||
with mock.patch.object(main, "pg_connect", no_db):
|
||||
with self.assertRaises(psycopg.OperationalError) as ctx:
|
||||
main._process_ready_batch(self.cfg, sets, s3)
|
||||
self.assertTrue(main.is_transient_error(ctx.exception))
|
||||
|
||||
def test_run_once_parallel_raises_when_a_worker_hits_outage(self):
|
||||
cfg = make_cfg(self.tmp.name, parallel_workers=3)
|
||||
s3 = FakeS3(entrance_objects(*UUIDS))
|
||||
with self.fail_first_with(s3_unavailable()), contextlib.redirect_stdout(io.StringIO()):
|
||||
with self.assertRaises(ClientError):
|
||||
main.run_once(cfg, main.PollerStats(None), s3c=s3)
|
||||
self.assertEqual(sorted(self.processed), UUIDS) # die anderen Worker laufen zu Ende
|
||||
|
||||
|
||||
class S3ClientConfigTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_poller_client_has_explicit_timeouts_and_retries(self):
|
||||
conf = main.S3(self.cfg, main.s3_client_config(self.cfg)).client.meta.config
|
||||
self.assertEqual(conf.connect_timeout, 10)
|
||||
self.assertEqual(conf.read_timeout, 30)
|
||||
self.assertEqual(conf.retries.get("mode"), "standard")
|
||||
self.assertEqual(conf.retries.get("total_max_attempts"), 4)
|
||||
self.assertNotIn("max_attempts", conf.retries)
|
||||
self.assertEqual(conf.signature_version, "s3v4")
|
||||
self.assertEqual(conf.s3.get("addressing_style"), "path")
|
||||
|
||||
def test_default_client_unchanged_for_gundi_path(self):
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
# exakt wie vor der Aenderung gebaut
|
||||
before = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=self.cfg.s3_endpoint,
|
||||
aws_access_key_id=self.cfg.s3_access_key,
|
||||
aws_secret_access_key=self.cfg.s3_secret_key,
|
||||
config=Config(max_pool_connections=self.cfg.s3_max_pool_connections),
|
||||
).meta.config
|
||||
now = main.S3(self.cfg).client.meta.config
|
||||
for attr in ("connect_timeout", "read_timeout", "retries", "max_pool_connections",
|
||||
"signature_version", "s3", "region_name"):
|
||||
with self.subTest(attr=attr):
|
||||
self.assertEqual(getattr(now, attr), getattr(before, attr))
|
||||
self.assertEqual(now.retries, {"mode": "legacy"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend.
|
||||
|
||||
Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein:
|
||||
|
||||
complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar
|
||||
jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json)
|
||||
json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json)
|
||||
cat_only nur _categories.json
|
||||
foreign Name passt zu keinem ICU-Muster
|
||||
|
||||
Alter = Alter des juengsten Objekts der Gruppe (konservativ).
|
||||
|
||||
python -m tools.entrance_inventory # Zusammenfassung
|
||||
python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job
|
||||
python -m tools.entrance_inventory --list jpg_only --min-age-days 7
|
||||
python -m tools.entrance_inventory --db --csv /tmp/entrance.csv
|
||||
|
||||
Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from main import CAT_RE, S3, UUID_RE, load_config
|
||||
|
||||
TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"]
|
||||
AGE_BUCKETS = [
|
||||
("<1h", 3600),
|
||||
("1h-1d", 86400),
|
||||
("1-7d", 7 * 86400),
|
||||
("7-30d", 30 * 86400),
|
||||
(">30d", None),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Group:
|
||||
uuid: Optional[str]
|
||||
parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size)
|
||||
|
||||
@property
|
||||
def kind(self) -> str:
|
||||
if self.uuid is None:
|
||||
return "foreign"
|
||||
has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts
|
||||
if has_jpg and has_meta:
|
||||
return "complete"
|
||||
if has_jpg:
|
||||
return "jpg_only"
|
||||
if has_meta:
|
||||
return "json_only"
|
||||
return "cat_only"
|
||||
|
||||
@property
|
||||
def newest(self) -> datetime:
|
||||
return max(p[1] for p in self.parts.values())
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return sum(p[2] for p in self.parts.values())
|
||||
|
||||
@property
|
||||
def keys(self) -> list[str]:
|
||||
return [p[0] for p in self.parts.values()]
|
||||
|
||||
|
||||
def age_bucket(age_s: float) -> str:
|
||||
for name, limit in AGE_BUCKETS:
|
||||
if limit is None or age_s < limit:
|
||||
return name
|
||||
return AGE_BUCKETS[-1][0]
|
||||
|
||||
|
||||
def collect(s3: S3) -> list[Group]:
|
||||
groups: dict[str, Group] = {}
|
||||
foreign: list[Group] = []
|
||||
for obj in s3.list_entrance_objects():
|
||||
key = obj["Key"]
|
||||
name = key.split("/")[-1]
|
||||
entry = (key, obj["LastModified"], obj.get("Size", 0))
|
||||
m = UUID_RE.match(name)
|
||||
if m:
|
||||
typ = "jpg" if m.group(2) == "jpg" else "meta"
|
||||
groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry
|
||||
continue
|
||||
m = CAT_RE.match(name)
|
||||
if m:
|
||||
groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry
|
||||
continue
|
||||
foreign.append(Group(None, {"other": entry}))
|
||||
return list(groups.values()) + foreign
|
||||
|
||||
|
||||
def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]:
|
||||
import psycopg
|
||||
|
||||
out: dict[str, tuple] = {}
|
||||
with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur:
|
||||
cur.execute("SET TRANSACTION READ ONLY")
|
||||
for i in range(0, len(uuids), 5000):
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200)
|
||||
FROM remote_cam.import_job
|
||||
WHERE image_uuid = ANY(%s)
|
||||
""",
|
||||
(uuids[i:i + 5000],),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
out[row[0]] = row[1:]
|
||||
return out
|
||||
|
||||
|
||||
def fmt_ts(dt: Optional[datetime]) -> str:
|
||||
return dt.strftime("%Y-%m-%d %H:%M") if dt else "-"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)")
|
||||
ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen")
|
||||
ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben")
|
||||
ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist")
|
||||
ap.add_argument("--csv", help="alle Gruppen als CSV schreiben")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
now = datetime.now(timezone.utc)
|
||||
groups = collect(S3(cfg))
|
||||
|
||||
status: dict[str, tuple] = {}
|
||||
if args.db:
|
||||
status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid])
|
||||
|
||||
min_age_s = args.min_age_days * 86400
|
||||
selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s]
|
||||
|
||||
# --- Zusammenfassung: Typ x Altersklasse
|
||||
by_kind: dict[str, list[Group]] = defaultdict(list)
|
||||
for g in selected:
|
||||
by_kind[g.kind].append(g)
|
||||
|
||||
bucket_names = [b[0] for b in AGE_BUCKETS]
|
||||
print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC"
|
||||
f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen"
|
||||
+ (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")")
|
||||
print()
|
||||
header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for kind in TYPES:
|
||||
gs = by_kind.get(kind, [])
|
||||
if not gs:
|
||||
continue
|
||||
buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs)
|
||||
mb = sum(g.size for g in gs) / 1e6
|
||||
print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names)
|
||||
+ f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}")
|
||||
|
||||
with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only")
|
||||
if with_cat:
|
||||
print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items())))
|
||||
|
||||
if args.db:
|
||||
print("\nimport_job-Status je Typ:")
|
||||
for kind in TYPES[:-1]:
|
||||
gs = by_kind.get(kind, [])
|
||||
if gs:
|
||||
c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs)
|
||||
print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common()))
|
||||
|
||||
# Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang.
|
||||
if by_kind.get("foreign"):
|
||||
print("\nFremde Objekte:")
|
||||
for g in sorted(by_kind["foreign"], key=lambda x: x.newest):
|
||||
key, lm, size = g.parts["other"]
|
||||
print(f" {fmt_ts(lm)} {size:>10} B {key}")
|
||||
|
||||
if args.list:
|
||||
print(f"\n# Keys vom Typ {args.list}:")
|
||||
for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest):
|
||||
for key in sorted(g.keys):
|
||||
print(key)
|
||||
|
||||
if args.csv:
|
||||
with open(args.csv, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes",
|
||||
"job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"])
|
||||
for g in sorted(selected, key=lambda x: (x.kind, x.newest)):
|
||||
st = status.get(g.uuid) if g.uuid else None
|
||||
w.writerow([
|
||||
g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts,
|
||||
g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size,
|
||||
st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "",
|
||||
st[3] if st else "", " ".join(sorted(g.keys)),
|
||||
])
|
||||
print(f"\nCSV geschrieben: {args.csv}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user