The poller crashed on every S3 or DB error (1162 restarts during the fsn1 object storage degradation on 2026-09-21). Now: - main loop catches errors per cycle, classifies them (S3 5xx/throttling, connection/timeouts, psycopg.OperationalError = transient; everything else logged with traceback) and retries with exponential backoff (5 s .. 5 min, reset after success); db_init runs until it succeeds once - explicit botocore config for the poller: connect/read timeouts, standard retries (total attempts), s3v4, path addressing; S3(cfg) without a config keeps the previous client for the Gundi path - DB connect timeout + TCP keepalives - one S3 client per cycle shared by all workers - a batch stops after a transient S3/DB error (image stays ERROR as before) - no busy loop when ready sets keep failing - [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck - tools/entrance_inventory.py: read-only listing of icu/entrance orphans - unit tests with simulated S3/DB failures Per-image transaction, status names/order, keys and paths are unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
210 lines
7.9 KiB
Python
210 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend.
|
|
|
|
Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein:
|
|
|
|
complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar
|
|
jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json)
|
|
json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json)
|
|
cat_only nur _categories.json
|
|
foreign Name passt zu keinem ICU-Muster
|
|
|
|
Alter = Alter des juengsten Objekts der Gruppe (konservativ).
|
|
|
|
python -m tools.entrance_inventory # Zusammenfassung
|
|
python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job
|
|
python -m tools.entrance_inventory --list jpg_only --min-age-days 7
|
|
python -m tools.entrance_inventory --db --csv /tmp/entrance.csv
|
|
|
|
Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import os
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
from main import CAT_RE, S3, UUID_RE, load_config
|
|
|
|
TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"]
|
|
AGE_BUCKETS = [
|
|
("<1h", 3600),
|
|
("1h-1d", 86400),
|
|
("1-7d", 7 * 86400),
|
|
("7-30d", 30 * 86400),
|
|
(">30d", None),
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class Group:
|
|
uuid: Optional[str]
|
|
parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size)
|
|
|
|
@property
|
|
def kind(self) -> str:
|
|
if self.uuid is None:
|
|
return "foreign"
|
|
has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts
|
|
if has_jpg and has_meta:
|
|
return "complete"
|
|
if has_jpg:
|
|
return "jpg_only"
|
|
if has_meta:
|
|
return "json_only"
|
|
return "cat_only"
|
|
|
|
@property
|
|
def newest(self) -> datetime:
|
|
return max(p[1] for p in self.parts.values())
|
|
|
|
@property
|
|
def size(self) -> int:
|
|
return sum(p[2] for p in self.parts.values())
|
|
|
|
@property
|
|
def keys(self) -> list[str]:
|
|
return [p[0] for p in self.parts.values()]
|
|
|
|
|
|
def age_bucket(age_s: float) -> str:
|
|
for name, limit in AGE_BUCKETS:
|
|
if limit is None or age_s < limit:
|
|
return name
|
|
return AGE_BUCKETS[-1][0]
|
|
|
|
|
|
def collect(s3: S3) -> list[Group]:
|
|
groups: dict[str, Group] = {}
|
|
foreign: list[Group] = []
|
|
for obj in s3.list_entrance_objects():
|
|
key = obj["Key"]
|
|
name = key.split("/")[-1]
|
|
entry = (key, obj["LastModified"], obj.get("Size", 0))
|
|
m = UUID_RE.match(name)
|
|
if m:
|
|
typ = "jpg" if m.group(2) == "jpg" else "meta"
|
|
groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry
|
|
continue
|
|
m = CAT_RE.match(name)
|
|
if m:
|
|
groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry
|
|
continue
|
|
foreign.append(Group(None, {"other": entry}))
|
|
return list(groups.values()) + foreign
|
|
|
|
|
|
def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]:
|
|
import psycopg
|
|
|
|
out: dict[str, tuple] = {}
|
|
with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur:
|
|
cur.execute("SET TRANSACTION READ ONLY")
|
|
for i in range(0, len(uuids), 5000):
|
|
cur.execute(
|
|
"""
|
|
SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200)
|
|
FROM remote_cam.import_job
|
|
WHERE image_uuid = ANY(%s)
|
|
""",
|
|
(uuids[i:i + 5000],),
|
|
)
|
|
for row in cur.fetchall():
|
|
out[row[0]] = row[1:]
|
|
return out
|
|
|
|
|
|
def fmt_ts(dt: Optional[datetime]) -> str:
|
|
return dt.strftime("%Y-%m-%d %H:%M") if dt else "-"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)")
|
|
ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen")
|
|
ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben")
|
|
ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist")
|
|
ap.add_argument("--csv", help="alle Gruppen als CSV schreiben")
|
|
args = ap.parse_args()
|
|
|
|
cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
|
now = datetime.now(timezone.utc)
|
|
groups = collect(S3(cfg))
|
|
|
|
status: dict[str, tuple] = {}
|
|
if args.db:
|
|
status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid])
|
|
|
|
min_age_s = args.min_age_days * 86400
|
|
selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s]
|
|
|
|
# --- Zusammenfassung: Typ x Altersklasse
|
|
by_kind: dict[str, list[Group]] = defaultdict(list)
|
|
for g in selected:
|
|
by_kind[g.kind].append(g)
|
|
|
|
bucket_names = [b[0] for b in AGE_BUCKETS]
|
|
print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC"
|
|
f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen"
|
|
+ (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")")
|
|
print()
|
|
header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes"
|
|
print(header)
|
|
print("-" * len(header))
|
|
for kind in TYPES:
|
|
gs = by_kind.get(kind, [])
|
|
if not gs:
|
|
continue
|
|
buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs)
|
|
mb = sum(g.size for g in gs) / 1e6
|
|
print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names)
|
|
+ f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}")
|
|
|
|
with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only")
|
|
if with_cat:
|
|
print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items())))
|
|
|
|
if args.db:
|
|
print("\nimport_job-Status je Typ:")
|
|
for kind in TYPES[:-1]:
|
|
gs = by_kind.get(kind, [])
|
|
if gs:
|
|
c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs)
|
|
print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common()))
|
|
|
|
# Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang.
|
|
if by_kind.get("foreign"):
|
|
print("\nFremde Objekte:")
|
|
for g in sorted(by_kind["foreign"], key=lambda x: x.newest):
|
|
key, lm, size = g.parts["other"]
|
|
print(f" {fmt_ts(lm)} {size:>10} B {key}")
|
|
|
|
if args.list:
|
|
print(f"\n# Keys vom Typ {args.list}:")
|
|
for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest):
|
|
for key in sorted(g.keys):
|
|
print(key)
|
|
|
|
if args.csv:
|
|
with open(args.csv, "w", newline="", encoding="utf-8") as f:
|
|
w = csv.writer(f)
|
|
w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes",
|
|
"job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"])
|
|
for g in sorted(selected, key=lambda x: (x.kind, x.newest)):
|
|
st = status.get(g.uuid) if g.uuid else None
|
|
w.writerow([
|
|
g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts,
|
|
g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size,
|
|
st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "",
|
|
st[3] if st else "", " ".join(sorted(g.keys)),
|
|
])
|
|
print(f"\nCSV geschrieben: {args.csv}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|