Make legacy poller resilient to S3/DB outages

The poller crashed on every S3 or DB error (1162 restarts during the
fsn1 object storage degradation on 2026-09-21). Now:

- main loop catches errors per cycle, classifies them (S3 5xx/throttling,
  connection/timeouts, psycopg.OperationalError = transient; everything
  else logged with traceback) and retries with exponential backoff
  (5 s .. 5 min, reset after success); db_init runs until it succeeds once
- explicit botocore config for the poller: connect/read timeouts, standard
  retries (total attempts), s3v4, path addressing; S3(cfg) without a
  config keeps the previous client for the Gundi path
- DB connect timeout + TCP keepalives
- one S3 client per cycle shared by all workers
- a batch stops after a transient S3/DB error (image stays ERROR as before)
- no busy loop when ready sets keep failing
- [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck
- tools/entrance_inventory.py: read-only listing of icu/entrance orphans
- unit tests with simulated S3/DB failures

Per-image transaction, status names/order, keys and paths are unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-09-21 16:56:55 +02:00
co-authored by Claude Opus 5
parent 2fc8dea10c
commit 0f574cccde
6 changed files with 1023 additions and 18 deletions
+209
View File
@@ -0,0 +1,209 @@
#!/usr/bin/env python3
"""Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend.
Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein:
complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar
jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json)
json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json)
cat_only nur _categories.json
foreign Name passt zu keinem ICU-Muster
Alter = Alter des juengsten Objekts der Gruppe (konservativ).
python -m tools.entrance_inventory # Zusammenfassung
python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job
python -m tools.entrance_inventory --list jpg_only --min-age-days 7
python -m tools.entrance_inventory --db --csv /tmp/entrance.csv
Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts.
"""
import argparse
import csv
import os
import sys
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Optional
from main import CAT_RE, S3, UUID_RE, load_config
TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"]
AGE_BUCKETS = [
("<1h", 3600),
("1h-1d", 86400),
("1-7d", 7 * 86400),
("7-30d", 30 * 86400),
(">30d", None),
]
@dataclass
class Group:
uuid: Optional[str]
parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size)
@property
def kind(self) -> str:
if self.uuid is None:
return "foreign"
has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts
if has_jpg and has_meta:
return "complete"
if has_jpg:
return "jpg_only"
if has_meta:
return "json_only"
return "cat_only"
@property
def newest(self) -> datetime:
return max(p[1] for p in self.parts.values())
@property
def size(self) -> int:
return sum(p[2] for p in self.parts.values())
@property
def keys(self) -> list[str]:
return [p[0] for p in self.parts.values()]
def age_bucket(age_s: float) -> str:
for name, limit in AGE_BUCKETS:
if limit is None or age_s < limit:
return name
return AGE_BUCKETS[-1][0]
def collect(s3: S3) -> list[Group]:
groups: dict[str, Group] = {}
foreign: list[Group] = []
for obj in s3.list_entrance_objects():
key = obj["Key"]
name = key.split("/")[-1]
entry = (key, obj["LastModified"], obj.get("Size", 0))
m = UUID_RE.match(name)
if m:
typ = "jpg" if m.group(2) == "jpg" else "meta"
groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry
continue
m = CAT_RE.match(name)
if m:
groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry
continue
foreign.append(Group(None, {"other": entry}))
return list(groups.values()) + foreign
def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]:
import psycopg
out: dict[str, tuple] = {}
with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur:
cur.execute("SET TRANSACTION READ ONLY")
for i in range(0, len(uuids), 5000):
cur.execute(
"""
SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200)
FROM remote_cam.import_job
WHERE image_uuid = ANY(%s)
""",
(uuids[i:i + 5000],),
)
for row in cur.fetchall():
out[row[0]] = row[1:]
return out
def fmt_ts(dt: Optional[datetime]) -> str:
return dt.strftime("%Y-%m-%d %H:%M") if dt else "-"
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)")
ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen")
ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben")
ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist")
ap.add_argument("--csv", help="alle Gruppen als CSV schreiben")
args = ap.parse_args()
cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml"))
now = datetime.now(timezone.utc)
groups = collect(S3(cfg))
status: dict[str, tuple] = {}
if args.db:
status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid])
min_age_s = args.min_age_days * 86400
selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s]
# --- Zusammenfassung: Typ x Altersklasse
by_kind: dict[str, list[Group]] = defaultdict(list)
for g in selected:
by_kind[g.kind].append(g)
bucket_names = [b[0] for b in AGE_BUCKETS]
print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC"
f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen"
+ (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")")
print()
header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes"
print(header)
print("-" * len(header))
for kind in TYPES:
gs = by_kind.get(kind, [])
if not gs:
continue
buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs)
mb = sum(g.size for g in gs) / 1e6
print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names)
+ f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}")
with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only")
if with_cat:
print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items())))
if args.db:
print("\nimport_job-Status je Typ:")
for kind in TYPES[:-1]:
gs = by_kind.get(kind, [])
if gs:
c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs)
print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common()))
# Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang.
if by_kind.get("foreign"):
print("\nFremde Objekte:")
for g in sorted(by_kind["foreign"], key=lambda x: x.newest):
key, lm, size = g.parts["other"]
print(f" {fmt_ts(lm)} {size:>10} B {key}")
if args.list:
print(f"\n# Keys vom Typ {args.list}:")
for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest):
for key in sorted(g.keys):
print(key)
if args.csv:
with open(args.csv, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes",
"job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"])
for g in sorted(selected, key=lambda x: (x.kind, x.newest)):
st = status.get(g.uuid) if g.uuid else None
w.writerow([
g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts,
g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size,
st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "",
st[3] if st else "", " ".join(sorted(g.keys)),
])
print(f"\nCSV geschrieben: {args.csv}")
return 0
if __name__ == "__main__":
sys.exit(main())