Make legacy poller resilient to S3/DB outages
The poller crashed on every S3 or DB error (1162 restarts during the fsn1 object storage degradation on 2026-09-21). Now: - main loop catches errors per cycle, classifies them (S3 5xx/throttling, connection/timeouts, psycopg.OperationalError = transient; everything else logged with traceback) and retries with exponential backoff (5 s .. 5 min, reset after success); db_init runs until it succeeds once - explicit botocore config for the poller: connect/read timeouts, standard retries (total attempts), s3v4, path addressing; S3(cfg) without a config keeps the previous client for the Gundi path - DB connect timeout + TCP keepalives - one S3 client per cycle shared by all workers - a batch stops after a transient S3/DB error (image stays ERROR as before) - no busy loop when ready sets keep failing - [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck - tools/entrance_inventory.py: read-only listing of icu/entrance orphans - unit tests with simulated S3/DB failures Per-image transaction, status names/order, keys and paths are unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend.
|
||||
|
||||
Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein:
|
||||
|
||||
complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar
|
||||
jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json)
|
||||
json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json)
|
||||
cat_only nur _categories.json
|
||||
foreign Name passt zu keinem ICU-Muster
|
||||
|
||||
Alter = Alter des juengsten Objekts der Gruppe (konservativ).
|
||||
|
||||
python -m tools.entrance_inventory # Zusammenfassung
|
||||
python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job
|
||||
python -m tools.entrance_inventory --list jpg_only --min-age-days 7
|
||||
python -m tools.entrance_inventory --db --csv /tmp/entrance.csv
|
||||
|
||||
Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from main import CAT_RE, S3, UUID_RE, load_config
|
||||
|
||||
TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"]
|
||||
AGE_BUCKETS = [
|
||||
("<1h", 3600),
|
||||
("1h-1d", 86400),
|
||||
("1-7d", 7 * 86400),
|
||||
("7-30d", 30 * 86400),
|
||||
(">30d", None),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Group:
|
||||
uuid: Optional[str]
|
||||
parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size)
|
||||
|
||||
@property
|
||||
def kind(self) -> str:
|
||||
if self.uuid is None:
|
||||
return "foreign"
|
||||
has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts
|
||||
if has_jpg and has_meta:
|
||||
return "complete"
|
||||
if has_jpg:
|
||||
return "jpg_only"
|
||||
if has_meta:
|
||||
return "json_only"
|
||||
return "cat_only"
|
||||
|
||||
@property
|
||||
def newest(self) -> datetime:
|
||||
return max(p[1] for p in self.parts.values())
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return sum(p[2] for p in self.parts.values())
|
||||
|
||||
@property
|
||||
def keys(self) -> list[str]:
|
||||
return [p[0] for p in self.parts.values()]
|
||||
|
||||
|
||||
def age_bucket(age_s: float) -> str:
|
||||
for name, limit in AGE_BUCKETS:
|
||||
if limit is None or age_s < limit:
|
||||
return name
|
||||
return AGE_BUCKETS[-1][0]
|
||||
|
||||
|
||||
def collect(s3: S3) -> list[Group]:
|
||||
groups: dict[str, Group] = {}
|
||||
foreign: list[Group] = []
|
||||
for obj in s3.list_entrance_objects():
|
||||
key = obj["Key"]
|
||||
name = key.split("/")[-1]
|
||||
entry = (key, obj["LastModified"], obj.get("Size", 0))
|
||||
m = UUID_RE.match(name)
|
||||
if m:
|
||||
typ = "jpg" if m.group(2) == "jpg" else "meta"
|
||||
groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry
|
||||
continue
|
||||
m = CAT_RE.match(name)
|
||||
if m:
|
||||
groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry
|
||||
continue
|
||||
foreign.append(Group(None, {"other": entry}))
|
||||
return list(groups.values()) + foreign
|
||||
|
||||
|
||||
def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]:
|
||||
import psycopg
|
||||
|
||||
out: dict[str, tuple] = {}
|
||||
with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur:
|
||||
cur.execute("SET TRANSACTION READ ONLY")
|
||||
for i in range(0, len(uuids), 5000):
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200)
|
||||
FROM remote_cam.import_job
|
||||
WHERE image_uuid = ANY(%s)
|
||||
""",
|
||||
(uuids[i:i + 5000],),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
out[row[0]] = row[1:]
|
||||
return out
|
||||
|
||||
|
||||
def fmt_ts(dt: Optional[datetime]) -> str:
|
||||
return dt.strftime("%Y-%m-%d %H:%M") if dt else "-"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)")
|
||||
ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen")
|
||||
ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben")
|
||||
ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist")
|
||||
ap.add_argument("--csv", help="alle Gruppen als CSV schreiben")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
now = datetime.now(timezone.utc)
|
||||
groups = collect(S3(cfg))
|
||||
|
||||
status: dict[str, tuple] = {}
|
||||
if args.db:
|
||||
status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid])
|
||||
|
||||
min_age_s = args.min_age_days * 86400
|
||||
selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s]
|
||||
|
||||
# --- Zusammenfassung: Typ x Altersklasse
|
||||
by_kind: dict[str, list[Group]] = defaultdict(list)
|
||||
for g in selected:
|
||||
by_kind[g.kind].append(g)
|
||||
|
||||
bucket_names = [b[0] for b in AGE_BUCKETS]
|
||||
print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC"
|
||||
f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen"
|
||||
+ (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")")
|
||||
print()
|
||||
header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for kind in TYPES:
|
||||
gs = by_kind.get(kind, [])
|
||||
if not gs:
|
||||
continue
|
||||
buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs)
|
||||
mb = sum(g.size for g in gs) / 1e6
|
||||
print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names)
|
||||
+ f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}")
|
||||
|
||||
with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only")
|
||||
if with_cat:
|
||||
print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items())))
|
||||
|
||||
if args.db:
|
||||
print("\nimport_job-Status je Typ:")
|
||||
for kind in TYPES[:-1]:
|
||||
gs = by_kind.get(kind, [])
|
||||
if gs:
|
||||
c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs)
|
||||
print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common()))
|
||||
|
||||
# Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang.
|
||||
if by_kind.get("foreign"):
|
||||
print("\nFremde Objekte:")
|
||||
for g in sorted(by_kind["foreign"], key=lambda x: x.newest):
|
||||
key, lm, size = g.parts["other"]
|
||||
print(f" {fmt_ts(lm)} {size:>10} B {key}")
|
||||
|
||||
if args.list:
|
||||
print(f"\n# Keys vom Typ {args.list}:")
|
||||
for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest):
|
||||
for key in sorted(g.keys):
|
||||
print(key)
|
||||
|
||||
if args.csv:
|
||||
with open(args.csv, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes",
|
||||
"job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"])
|
||||
for g in sorted(selected, key=lambda x: (x.kind, x.newest)):
|
||||
st = status.get(g.uuid) if g.uuid else None
|
||||
w.writerow([
|
||||
g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts,
|
||||
g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size,
|
||||
st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "",
|
||||
st[3] if st else "", " ".join(sorted(g.keys)),
|
||||
])
|
||||
print(f"\nCSV geschrieben: {args.csv}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user