#!/usr/bin/env python3 """Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend. Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein: complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json) json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json) cat_only nur _categories.json foreign Name passt zu keinem ICU-Muster Alter = Alter des juengsten Objekts der Gruppe (konservativ). python -m tools.entrance_inventory # Zusammenfassung python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job python -m tools.entrance_inventory --list jpg_only --min-age-days 7 python -m tools.entrance_inventory --db --csv /tmp/entrance.csv Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts. """ import argparse import csv import os import sys from collections import Counter, defaultdict from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Optional from main import CAT_RE, S3, UUID_RE, load_config TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"] AGE_BUCKETS = [ ("<1h", 3600), ("1h-1d", 86400), ("1-7d", 7 * 86400), ("7-30d", 30 * 86400), (">30d", None), ] @dataclass class Group: uuid: Optional[str] parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size) @property def kind(self) -> str: if self.uuid is None: return "foreign" has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts if has_jpg and has_meta: return "complete" if has_jpg: return "jpg_only" if has_meta: return "json_only" return "cat_only" @property def newest(self) -> datetime: return max(p[1] for p in self.parts.values()) @property def size(self) -> int: return sum(p[2] for p in self.parts.values()) @property def keys(self) -> list[str]: return [p[0] for p in self.parts.values()] def age_bucket(age_s: float) -> str: for name, limit in AGE_BUCKETS: if limit is None or age_s < limit: return name return AGE_BUCKETS[-1][0] def collect(s3: S3) -> list[Group]: groups: dict[str, Group] = {} foreign: list[Group] = [] for obj in s3.list_entrance_objects(): key = obj["Key"] name = key.split("/")[-1] entry = (key, obj["LastModified"], obj.get("Size", 0)) m = UUID_RE.match(name) if m: typ = "jpg" if m.group(2) == "jpg" else "meta" groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry continue m = CAT_RE.match(name) if m: groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry continue foreign.append(Group(None, {"other": entry})) return list(groups.values()) + foreign def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]: import psycopg out: dict[str, tuple] = {} with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur: cur.execute("SET TRANSACTION READ ONLY") for i in range(0, len(uuids), 5000): cur.execute( """ SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200) FROM remote_cam.import_job WHERE image_uuid = ANY(%s) """, (uuids[i:i + 5000],), ) for row in cur.fetchall(): out[row[0]] = row[1:] return out def fmt_ts(dt: Optional[datetime]) -> str: return dt.strftime("%Y-%m-%d %H:%M") if dt else "-" def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)") ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen") ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben") ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist") ap.add_argument("--csv", help="alle Gruppen als CSV schreiben") args = ap.parse_args() cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml")) now = datetime.now(timezone.utc) groups = collect(S3(cfg)) status: dict[str, tuple] = {} if args.db: status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid]) min_age_s = args.min_age_days * 86400 selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s] # --- Zusammenfassung: Typ x Altersklasse by_kind: dict[str, list[Group]] = defaultdict(list) for g in selected: by_kind[g.kind].append(g) bucket_names = [b[0] for b in AGE_BUCKETS] print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC" f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen" + (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")") print() header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes" print(header) print("-" * len(header)) for kind in TYPES: gs = by_kind.get(kind, []) if not gs: continue buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs) mb = sum(g.size for g in gs) / 1e6 print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names) + f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}") with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only") if with_cat: print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items()))) if args.db: print("\nimport_job-Status je Typ:") for kind in TYPES[:-1]: gs = by_kind.get(kind, []) if gs: c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs) print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common())) # Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang. if by_kind.get("foreign"): print("\nFremde Objekte:") for g in sorted(by_kind["foreign"], key=lambda x: x.newest): key, lm, size = g.parts["other"] print(f" {fmt_ts(lm)} {size:>10} B {key}") if args.list: print(f"\n# Keys vom Typ {args.list}:") for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest): for key in sorted(g.keys): print(key) if args.csv: with open(args.csv, "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes", "job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"]) for g in sorted(selected, key=lambda x: (x.kind, x.newest)): st = status.get(g.uuid) if g.uuid else None w.writerow([ g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts, g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size, st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "", st[3] if st else "", " ".join(sorted(g.keys)), ]) print(f"\nCSV geschrieben: {args.csv}") return 0 if __name__ == "__main__": sys.exit(main())