From ed105fbff8174e4ea7245ff04c671b9c2802ab14 Mon Sep 17 00:00:00 2001 From: Dominik Dachs Date: Fri, 16 Oct 2026 21:01:17 +0200 Subject: [PATCH] Prepare legacy migration for the portal cutover - detection from legacy category only for scores >= 0.5, without ''/none - category_label + view detection_labeled unify ICU and legacy labels - schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images (same imei, same size, <= 10 min), archived in image_superseded - migrate_legacy --plan: read-only preview of the migration - Gundi S3 prefix icu/gundi-test/ -> icu/gundi/ Co-Authored-By: Claude Opus 5.5 --- README.md | 29 +++++--- config.yaml | 6 +- gundi_pipeline.py | 6 +- schema/001_domain_model.sql | 34 ++++++++++ schema/002_migrate_legacy.sql | 6 +- schema/003_dedupe_gundi_twins.sql | 43 ++++++++++++ tests/test_gundi_resilience.py | 4 +- tests/test_migrate_legacy.py | 93 +++++++++++++++++++++++++ tools/migrate_legacy.py | 108 ++++++++++++++++++++++++++++-- 9 files changed, 307 insertions(+), 22 deletions(-) create mode 100644 schema/003_dedupe_gundi_twins.sql create mode 100644 tests/test_migrate_legacy.py diff --git a/README.md b/README.md index 858d465..bb48307 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,9 @@ Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne W |---------|--------| | `remote_cam.camera_status` | Statusupdates / Kameraeigenschaften (Zeitreihe je IMEI) | | `remote_cam.image` | Bildeigenschaften inkl. `servertime`, `ocr_ts`, `ocr_state` | -| `remote_cam.detection` | Objektklassifikationen, `confidence` 0..1 | +| `remote_cam.detection` | Objektklassifikationen, `confidence` 0..1, Rohbezeichnung der Quelle | +| `remote_cam.category_label` / View `detection_labeled` | einheitliches Label je Rohbezeichnung (`Deer-Like` -> `deer`) | +| `remote_cam.image_superseded` | entfernte Gundi-Zwillinge aus dem Parallelbetrieb (mit Originalzeile) | | `remote_cam.image_with_status` | View: Bild + zum `servertime` gueltiger Status | | `remote_cam.gundi_inbox` | Transportschicht: Rohevents + Zustand des Push-Pfads | @@ -124,15 +126,26 @@ Das Schema legt der Receiver beim Start an. Die Legacy-Tabellen `metadata` / ### Bestandsdaten migrieren -Split `metadata` -> `image` + `camera_status`, `category` -> `detection`. -Wiederholbar, batchweise, liest die Alttabellen nur: +Split `metadata` -> `image` + `camera_status`, `category` -> `detection` +(nur Klassen ab Score 0.5, ohne `''`/`none`). Wiederholbar, batchweise, liest die +Alttabellen nur: ```bash +docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --plan # nur lesen docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy ``` -Vor dem Portal-Cutover erneut ausfuehren, um zwischenzeitlich neue Bilder -nachzuziehen. `--verify-only` prueft nur die Zaehlungen. +Im Parallelbetrieb kommt dasselbe Bild auch ueber Gundi (gleiche imei, gleiche +Dateigroesse, <= 10 min). Nach der Migration den Gundi-Zwilling entfernen; behalten +wird der Legacy-Zwilling, an dem die KI-Ergebnisse haengen: + +```bash +docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --dedupe --dry-run +docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --dedupe +``` + +Vor dem Portal-Cutover und nach dem Stoppen des Pollers beides erneut ausfuehren. +`--verify-only` prueft nur die Zaehlungen. ### Test @@ -142,9 +155,9 @@ docker compose run --rm --no-deps gundi-receiver \ python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg /pfad/zu/bild.jpg ``` -Standardmaessig schreibt der Pfad unter `icu/gundi-test/` (siehe `gundi:` in -`config.yaml`); fuer den Produktivbetrieb `processed_prefix` / `thumb_prefix` auf -`icu/processed/` / `icu/thumbnails/` umstellen. +Der Pfad schreibt unter `icu/gundi/` (siehe `gundi:` in `config.yaml`); bis +2026-10-16 war es `icu/gundi-test/`. Der genaue Schluessel steht je Bild in +`image.processed_key` / `thumbnail_key` - Konsumenten bauen ihn nicht selbst zusammen. ### nginx (`icu.meles.eu`) diff --git a/config.yaml b/config.yaml index b689b40..9547fe1 100644 --- a/config.yaml +++ b/config.yaml @@ -55,9 +55,9 @@ gundi: enabled: true bind_host: "0.0.0.0" # im Container; nach aussen nur 127.0.0.1:8090 (compose) bind_port: 8090 - inbox_prefix: "icu/gundi-test/inbox/" - processed_prefix: "icu/gundi-test/processed/" # spaeter: icu/processed/ - thumb_prefix: "icu/gundi-test/thumbnails/" # spaeter: icu/thumbnails/ + inbox_prefix: "icu/gundi/inbox/" + processed_prefix: "icu/gundi/processed/" # Portal liest den Schluessel aus image.processed_key + thumb_prefix: "icu/gundi/thumbnails/" # Portal liest den Schluessel aus image.thumbnail_key workers: 4 ocr_workers: 2 max_request_bytes: 33554432 diff --git a/gundi_pipeline.py b/gundi_pipeline.py index d15e310..0d6f10e 100644 --- a/gundi_pipeline.py +++ b/gundi_pipeline.py @@ -47,9 +47,9 @@ class GundiConfig: api_key: str bind_host: str = "0.0.0.0" bind_port: int = 8090 - inbox_prefix: str = "icu/gundi-test/inbox/" - processed_prefix: str = "icu/gundi-test/processed/" - thumb_prefix: str = "icu/gundi-test/thumbnails/" + inbox_prefix: str = "icu/gundi/inbox/" + processed_prefix: str = "icu/gundi/processed/" + thumb_prefix: str = "icu/gundi/thumbnails/" workers: int = 4 ocr_workers: int = 2 max_request_bytes: int = 32 * 1024 * 1024 diff --git a/schema/001_domain_model.sql b/schema/001_domain_model.sql index 4ff592b..edc81b9 100644 --- a/schema/001_domain_model.sql +++ b/schema/001_domain_model.sql @@ -98,6 +98,40 @@ CREATE INDEX IF NOT EXISTS detection_image_confidence_idx CREATE INDEX IF NOT EXISTS detection_category_idx ON remote_cam.detection (category); +-- Einheitliches Label je Rohkategorie. detection.category bleibt die Rohbezeichnung der +-- Quelle (ICU-Gundi: "Deer-Like", Legacy-Klassifikator: "deer"); Auswertungen nutzen das Label. +CREATE TABLE IF NOT EXISTS remote_cam.category_label ( + category text PRIMARY KEY, + label text NOT NULL +); + +INSERT INTO remote_cam.category_label (category, label) VALUES + ('Deer-Like', 'deer'), ('deer', 'deer'), + ('person', 'human'), ('human', 'human'), + ('car', 'machine'), ('machine', 'machine'), + ('Bird', 'bird'), ('bird', 'bird'), + ('Fox', 'fox'), ('fox', 'fox'), + ('Wild Boar', 'wild boar'), ('wild boar', 'wild boar'), + ('racoon', 'raccoon'), ('raccoon', 'raccoon') +ON CONFLICT (category) DO NOTHING; + +-- Ohne Eintrag in category_label gilt die kleingeschriebene Rohbezeichnung. +CREATE OR REPLACE VIEW remote_cam.detection_labeled AS +SELECT d.*, COALESCE(cl.label, lower(d.category)) AS label + FROM remote_cam.detection d + LEFT JOIN remote_cam.category_label cl ON cl.category = d.category; + +-- Gundi-Bilder, die waehrend des Parallelbetriebs auch ueber den Legacy-Pfad kamen und +-- zugunsten des Legacy-Zwillings (traegt die KI-Ergebnisse) entfernt wurden. Haelt die +-- geloeschte image-Zeile samt Detections fuer Nachvollziehbarkeit und Rollback. +CREATE TABLE IF NOT EXISTS remote_cam.image_superseded ( + image_uuid uuid PRIMARY KEY, + superseded_by uuid NOT NULL, + removed_at timestamptz NOT NULL DEFAULT now(), + image_row jsonb NOT NULL, + detections jsonb +); + -- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine. CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox ( object_id uuid PRIMARY KEY, diff --git a/schema/002_migrate_legacy.sql b/schema/002_migrate_legacy.sql index b14fa6f..a95df96 100644 --- a/schema/002_migrate_legacy.sql +++ b/schema/002_migrate_legacy.sql @@ -1,5 +1,5 @@ -- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt --- ein image und ein camera_status; remote_cam.category wird zu detection. +-- ein image und ein camera_status; remote_cam.category (Score >= 0.5) wird zu detection. -- -- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py -- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen. @@ -82,12 +82,16 @@ WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING; -- step: detection +-- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes: +-- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection). INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source) SELECT i.image_uuid, c.category, c.score, c.version, 'entrance' FROM _mig_batch b JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid WHERE c.category IS NOT NULL + AND c.category NOT IN ('', 'none') + AND c.score >= 0.5 AND NOT EXISTS ( SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance' diff --git a/schema/003_dedupe_gundi_twins.sql b/schema/003_dedupe_gundi_twins.sql new file mode 100644 index 0000000..5418374 --- /dev/null +++ b/schema/003_dedupe_gundi_twins.sql @@ -0,0 +1,43 @@ +-- Parallelbetrieb Legacy + Gundi: dasselbe Bild liegt doppelt in remote_cam.image. +-- Behalten wird der Legacy-Zwilling (source='entrance'): an seiner UUID haengen die +-- KI-Ergebnisse (ai_image/ai_object). Der Gundi-Zwilling wandert nach image_superseded. +-- +-- Zwilling = gleiche imei, gleiche Dateigroesse, servertime hoechstens 10 min auseinander. +-- Gemessen am 2026-10-16: 4291 Paare, jedes Bild hat hoechstens einen Partner. +-- Nur Paare, deren Legacy-Seite schon migriert ist; Paare mit mehreren Kandidaten bleiben +-- unangetastet. Wiederholbar. tools/migrate_legacy.py --dedupe fuehrt die Schritte in EINER +-- Transaktion aus. + +-- step: twins +CREATE TEMP TABLE _twins ON COMMIT DROP AS +WITH pairs AS ( + SELECT g.image_uuid AS gundi_uuid, e.image_uuid AS legacy_uuid + FROM remote_cam.image g + JOIN remote_cam.image e + ON e.source = 'entrance' + AND e.imei = g.imei + AND e.size_bytes = g.size_bytes + AND e.servertime BETWEEN g.servertime - interval '10 minutes' + AND g.servertime + interval '10 minutes' + WHERE g.source = 'gundi' +) +SELECT gundi_uuid, legacy_uuid + FROM (SELECT p.*, + count(*) OVER (PARTITION BY p.gundi_uuid) AS n_gundi, + count(*) OVER (PARTITION BY p.legacy_uuid) AS n_legacy + FROM pairs p) x + WHERE n_gundi = 1 AND n_legacy = 1; + +-- step: archive +INSERT INTO remote_cam.image_superseded (image_uuid, superseded_by, image_row, detections) +SELECT t.gundi_uuid, t.legacy_uuid, to_jsonb(i), + (SELECT jsonb_agg(to_jsonb(d)) FROM remote_cam.detection d WHERE d.image_uuid = t.gundi_uuid) + FROM _twins t + JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid +ON CONFLICT (image_uuid) DO NOTHING; + +-- step: delete +-- detection haengt per ON DELETE CASCADE am image. +DELETE FROM remote_cam.image i + USING _twins t + WHERE i.image_uuid = t.gundi_uuid; diff --git a/tests/test_gundi_resilience.py b/tests/test_gundi_resilience.py index 18090e2..2fc55d3 100644 --- a/tests/test_gundi_resilience.py +++ b/tests/test_gundi_resilience.py @@ -67,7 +67,7 @@ def fake_connect(cursor: FakeCursor): return lambda dsn: FakeConn(cursor) -GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3) +GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3) class AttachImageTests(unittest.TestCase): @@ -89,7 +89,7 @@ class AttachImageTests(unittest.TestCase): s3 = mock.Mock() with mock.patch.object(gp, "connect", fake_connect(cur)): self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored") - s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg") + s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg") self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0]) def test_already_received_is_duplicate_without_s3(self): diff --git a/tests/test_migrate_legacy.py b/tests/test_migrate_legacy.py new file mode 100644 index 0000000..667e0f4 --- /dev/null +++ b/tests/test_migrate_legacy.py @@ -0,0 +1,93 @@ +"""tools/migrate_legacy: Schrittdateien und --dedupe-Ablauf, ohne DB. + + python -m unittest discover -s tests -v +""" +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from tools import migrate_legacy as ml # noqa: E402 + + +class FakeCursor: + def __init__(self, n_twins, deleted=None): + self.executed = [] + self.n_twins = n_twins + self.deleted = n_twins if deleted is None else deleted + self.rowcount = 0 + self._next = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + self.executed.append(sql) + if "count(*) FROM _twins" in sql: + self._next = (self.n_twins,) + elif "min(i.servertime)" in sql: + self._next = (None, None) + elif sql.lstrip().startswith("INSERT INTO remote_cam.image_superseded"): + self.rowcount = self.n_twins + elif "DELETE FROM remote_cam.image" in sql: + self.rowcount = self.deleted + + def fetchone(self): + return self._next + + +def fake_conn(cur): + conn = mock.Mock() + conn.cursor.return_value = cur + return conn + + +class StepFiles(unittest.TestCase): + def test_migration_steps(self): + steps = dict(ml.load_steps()) + self.assertEqual(list(steps), ["image", "camera_status", "detection"]) + det = steps["detection"] + self.assertIn("c.score >= 0.5", det) + self.assertIn("NOT IN ('', 'none')", det) + + def test_dedupe_steps(self): + steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql")) + self.assertEqual(list(steps), ["twins", "archive", "delete"]) + self.assertIn("e.size_bytes = g.size_bytes", steps["twins"]) + self.assertIn("n_gundi = 1 AND n_legacy = 1", steps["twins"]) + + +class Dedupe(unittest.TestCase): + def test_dry_run_changes_nothing(self): + cur = FakeCursor(n_twins=5) + conn = fake_conn(cur) + ml.dedupe(conn, dry_run=True) + self.assertFalse(any("image_superseded" in s or "DELETE" in s for s in cur.executed)) + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + + def test_archives_then_deletes(self): + cur = FakeCursor(n_twins=5) + conn = fake_conn(cur) + ml.dedupe(conn, dry_run=False) + archive = next(i for i, s in enumerate(cur.executed) if "INSERT INTO remote_cam.image_superseded" in s) + delete = next(i for i, s in enumerate(cur.executed) if "DELETE FROM remote_cam.image" in s) + self.assertLess(archive, delete) + conn.commit.assert_called_once() + + def test_count_mismatch_rolls_back(self): + cur = FakeCursor(n_twins=5, deleted=4) + conn = fake_conn(cur) + with self.assertRaises(RuntimeError): + ml.dedupe(conn, dry_run=False) + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/migrate_legacy.py b/tools/migrate_legacy.py index a867353..c88dfad 100644 --- a/tools/migrate_legacy.py +++ b/tools/migrate_legacy.py @@ -4,8 +4,11 @@ python -m tools.migrate_legacy # PG_DSN aus der Umgebung python -m tools.migrate_legacy --dsn postgresql://... --batch 20000 python -m tools.migrate_legacy --verify-only + python -m tools.migrate_legacy --plan # nur lesen: was wuerde migriert/entfernt + python -m tools.migrate_legacy --dedupe --dry-run + python -m tools.migrate_legacy --dedupe # Gundi-Zwillinge entfernen (schema/003) -Wiederholbar. Jeder Batch ist eine eigene Transaktion. +Wiederholbar. Jeder Batch ist eine eigene Transaktion, --dedupe laeuft in einer. """ import argparse import os @@ -20,8 +23,8 @@ SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema" SCHEMA_LOCK_KEY = 7_311_000_001 -def load_steps() -> list[tuple[str, str]]: - sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8") +def load_steps(filename: str = "002_migrate_legacy.sql") -> list[tuple[str, str]]: + sql = (SCHEMA_DIR / filename).read_text(encoding="utf-8") parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE) return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)] @@ -74,6 +77,81 @@ def migrate(conn: psycopg.Connection, batch_size: int) -> None: print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True) +# Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge). +PLAN_QUERIES = [ + ("image(entrance) neu", + "SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " + "AND NOT EXISTS (SELECT 1 FROM remote_cam.image i WHERE i.image_uuid = m.metadata_uuid::uuid)"), + ("camera_status(entrance) neu", + "SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " + "AND NOT EXISTS (SELECT 1 FROM remote_cam.camera_status s WHERE s.source = 'entrance' " + "AND s.source_ref = m.metadata_uuid)"), + ("detection(entrance) neu", + "SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid " + "WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " + "AND c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND c.score >= 0.5 " + "AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid " + "AND d.source = 'entrance')"), + ("Gundi-Zwillinge (nach Migration zu entfernen)", + """ + WITH pairs AS ( + SELECT g.image_uuid AS gundi_uuid, m.metadata_uuid AS legacy_uuid + FROM remote_cam.image g + JOIN remote_cam.metadata m + ON m.imei = g.imei AND m."SizeInByte" = g.size_bytes + AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes' + WHERE g.source = 'gundi') + SELECT count(*) FROM ( + SELECT count(*) OVER (PARTITION BY gundi_uuid) AS n_g, count(*) OVER (PARTITION BY legacy_uuid) AS n_l + FROM pairs) x + WHERE n_g = 1 AND n_l = 1 + """), + ("Gundi-Bilder ohne Zwilling (bleiben)", + """ + SELECT count(*) FROM remote_cam.image g WHERE g.source = 'gundi' AND NOT EXISTS ( + SELECT 1 FROM remote_cam.metadata m + WHERE m.imei = g.imei AND m."SizeInByte" = g.size_bytes + AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes') + """), +] + + +def plan(conn: psycopg.Connection) -> None: + with conn.cursor() as cur: + cur.execute("SET TRANSACTION READ ONLY") + for label, sql in PLAN_QUERIES: + cur.execute(sql) + print(f" {label}: {cur.fetchone()[0]}", flush=True) + conn.rollback() + + +def dedupe(conn: psycopg.Connection, dry_run: bool) -> None: + steps = dict(load_steps("003_dedupe_gundi_twins.sql")) + with conn.cursor() as cur: + cur.execute(steps["twins"]) + cur.execute("SELECT count(*) FROM _twins") + n = cur.fetchone()[0] + cur.execute( + "SELECT min(i.servertime), max(i.servertime) FROM _twins t " + "JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid" + ) + lo, hi = cur.fetchone() + print(f" Gundi-Zwillinge: {n} (servertime {lo} .. {hi})", flush=True) + if dry_run: + conn.rollback() + print(" Trockenlauf: nichts geaendert.", flush=True) + return + cur.execute(steps["archive"]) + archived = cur.rowcount + cur.execute(steps["delete"]) + deleted = cur.rowcount + if deleted != n: + conn.rollback() + raise RuntimeError(f"geloescht={deleted} != Zwillinge={n}; zurueckgerollt") + conn.commit() + print(f" archiviert={archived} geloescht={deleted}", flush=True) + + def verify(conn: psycopg.Connection) -> bool: checks = [ ("metadata == image(entrance)", @@ -82,9 +160,16 @@ def verify(conn: psycopg.Connection) -> bool: ("metadata == camera_status(entrance)", "SELECT (SELECT count(*) FROM remote_cam.metadata), " "(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"), - ("category == detection(entrance)", - "SELECT (SELECT count(*) FROM remote_cam.category), " + ("category (Score >= 0.5, ohne ''/none) == detection(entrance)", + "SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m " + "ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " + "AND c.category NOT IN ('', 'none') AND c.score >= 0.5), " "(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"), + ("keine Gundi-Bilder mit Legacy-Zwilling", + "SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e " + "ON e.source = 'entrance' AND e.imei = g.imei AND e.size_bytes = g.size_bytes " + "AND e.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes' " + "WHERE g.source = 'gundi'"), ("metadata ohne ocr_ts == image FAIL", "SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), " "(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"), @@ -124,12 +209,25 @@ def main() -> int: ap.add_argument("--dsn", default=os.environ.get("PG_DSN")) ap.add_argument("--batch", type=int, default=20000) ap.add_argument("--verify-only", action="store_true") + ap.add_argument("--plan", action="store_true", help="nur lesen: Vorschau der Mengen") + ap.add_argument("--dedupe", action="store_true", help="Gundi-Zwillinge entfernen (schema/003)") + ap.add_argument("--dry-run", action="store_true", help="mit --dedupe: nur zaehlen") args = ap.parse_args() if not args.dsn: print("--dsn oder PG_DSN fehlt", file=sys.stderr) return 2 with psycopg.connect(args.dsn, connect_timeout=10) as conn: + if args.plan: + print("Vorschau (nur lesend):", flush=True) + plan(conn) + return 0 + if args.dedupe: + if not args.dry_run: + apply_schema(conn) + print("Gundi-Zwillinge:", flush=True) + dedupe(conn, args.dry_run) + return 0 if not args.verify_only: apply_schema(conn) print("Schema angewendet. Migriere ...", flush=True)