Prepare legacy migration for the portal cutover
- detection from legacy category only for scores >= 0.5, without ''/none - category_label + view detection_labeled unify ICU and legacy labels - schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images (same imei, same size, <= 10 min), archived in image_superseded - migrate_legacy --plan: read-only preview of the migration - Gundi S3 prefix icu/gundi-test/ -> icu/gundi/ Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -115,7 +115,9 @@ Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne W
|
|||||||
|---------|--------|
|
|---------|--------|
|
||||||
| `remote_cam.camera_status` | Statusupdates / Kameraeigenschaften (Zeitreihe je IMEI) |
|
| `remote_cam.camera_status` | Statusupdates / Kameraeigenschaften (Zeitreihe je IMEI) |
|
||||||
| `remote_cam.image` | Bildeigenschaften inkl. `servertime`, `ocr_ts`, `ocr_state` |
|
| `remote_cam.image` | Bildeigenschaften inkl. `servertime`, `ocr_ts`, `ocr_state` |
|
||||||
| `remote_cam.detection` | Objektklassifikationen, `confidence` 0..1 |
|
| `remote_cam.detection` | Objektklassifikationen, `confidence` 0..1, Rohbezeichnung der Quelle |
|
||||||
|
| `remote_cam.category_label` / View `detection_labeled` | einheitliches Label je Rohbezeichnung (`Deer-Like` -> `deer`) |
|
||||||
|
| `remote_cam.image_superseded` | entfernte Gundi-Zwillinge aus dem Parallelbetrieb (mit Originalzeile) |
|
||||||
| `remote_cam.image_with_status` | View: Bild + zum `servertime` gueltiger Status |
|
| `remote_cam.image_with_status` | View: Bild + zum `servertime` gueltiger Status |
|
||||||
| `remote_cam.gundi_inbox` | Transportschicht: Rohevents + Zustand des Push-Pfads |
|
| `remote_cam.gundi_inbox` | Transportschicht: Rohevents + Zustand des Push-Pfads |
|
||||||
|
|
||||||
@@ -124,15 +126,26 @@ Das Schema legt der Receiver beim Start an. Die Legacy-Tabellen `metadata` /
|
|||||||
|
|
||||||
### Bestandsdaten migrieren
|
### Bestandsdaten migrieren
|
||||||
|
|
||||||
Split `metadata` -> `image` + `camera_status`, `category` -> `detection`.
|
Split `metadata` -> `image` + `camera_status`, `category` -> `detection`
|
||||||
Wiederholbar, batchweise, liest die Alttabellen nur:
|
(nur Klassen ab Score 0.5, ohne `''`/`none`). Wiederholbar, batchweise, liest die
|
||||||
|
Alttabellen nur:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --plan # nur lesen
|
||||||
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy
|
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy
|
||||||
```
|
```
|
||||||
|
|
||||||
Vor dem Portal-Cutover erneut ausfuehren, um zwischenzeitlich neue Bilder
|
Im Parallelbetrieb kommt dasselbe Bild auch ueber Gundi (gleiche imei, gleiche
|
||||||
nachzuziehen. `--verify-only` prueft nur die Zaehlungen.
|
Dateigroesse, <= 10 min). Nach der Migration den Gundi-Zwilling entfernen; behalten
|
||||||
|
wird der Legacy-Zwilling, an dem die KI-Ergebnisse haengen:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --dedupe --dry-run
|
||||||
|
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --dedupe
|
||||||
|
```
|
||||||
|
|
||||||
|
Vor dem Portal-Cutover und nach dem Stoppen des Pollers beides erneut ausfuehren.
|
||||||
|
`--verify-only` prueft nur die Zaehlungen.
|
||||||
|
|
||||||
### Test
|
### Test
|
||||||
|
|
||||||
@@ -142,9 +155,9 @@ docker compose run --rm --no-deps gundi-receiver \
|
|||||||
python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg /pfad/zu/bild.jpg
|
python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg /pfad/zu/bild.jpg
|
||||||
```
|
```
|
||||||
|
|
||||||
Standardmaessig schreibt der Pfad unter `icu/gundi-test/` (siehe `gundi:` in
|
Der Pfad schreibt unter `icu/gundi/` (siehe `gundi:` in `config.yaml`); bis
|
||||||
`config.yaml`); fuer den Produktivbetrieb `processed_prefix` / `thumb_prefix` auf
|
2026-10-16 war es `icu/gundi-test/`. Der genaue Schluessel steht je Bild in
|
||||||
`icu/processed/` / `icu/thumbnails/` umstellen.
|
`image.processed_key` / `thumbnail_key` - Konsumenten bauen ihn nicht selbst zusammen.
|
||||||
|
|
||||||
### nginx (`icu.meles.eu`)
|
### nginx (`icu.meles.eu`)
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -55,9 +55,9 @@ gundi:
|
|||||||
enabled: true
|
enabled: true
|
||||||
bind_host: "0.0.0.0" # im Container; nach aussen nur 127.0.0.1:8090 (compose)
|
bind_host: "0.0.0.0" # im Container; nach aussen nur 127.0.0.1:8090 (compose)
|
||||||
bind_port: 8090
|
bind_port: 8090
|
||||||
inbox_prefix: "icu/gundi-test/inbox/"
|
inbox_prefix: "icu/gundi/inbox/"
|
||||||
processed_prefix: "icu/gundi-test/processed/" # spaeter: icu/processed/
|
processed_prefix: "icu/gundi/processed/" # Portal liest den Schluessel aus image.processed_key
|
||||||
thumb_prefix: "icu/gundi-test/thumbnails/" # spaeter: icu/thumbnails/
|
thumb_prefix: "icu/gundi/thumbnails/" # Portal liest den Schluessel aus image.thumbnail_key
|
||||||
workers: 4
|
workers: 4
|
||||||
ocr_workers: 2
|
ocr_workers: 2
|
||||||
max_request_bytes: 33554432
|
max_request_bytes: 33554432
|
||||||
|
|||||||
+3
-3
@@ -47,9 +47,9 @@ class GundiConfig:
|
|||||||
api_key: str
|
api_key: str
|
||||||
bind_host: str = "0.0.0.0"
|
bind_host: str = "0.0.0.0"
|
||||||
bind_port: int = 8090
|
bind_port: int = 8090
|
||||||
inbox_prefix: str = "icu/gundi-test/inbox/"
|
inbox_prefix: str = "icu/gundi/inbox/"
|
||||||
processed_prefix: str = "icu/gundi-test/processed/"
|
processed_prefix: str = "icu/gundi/processed/"
|
||||||
thumb_prefix: str = "icu/gundi-test/thumbnails/"
|
thumb_prefix: str = "icu/gundi/thumbnails/"
|
||||||
workers: int = 4
|
workers: int = 4
|
||||||
ocr_workers: int = 2
|
ocr_workers: int = 2
|
||||||
max_request_bytes: int = 32 * 1024 * 1024
|
max_request_bytes: int = 32 * 1024 * 1024
|
||||||
|
|||||||
@@ -98,6 +98,40 @@ CREATE INDEX IF NOT EXISTS detection_image_confidence_idx
|
|||||||
CREATE INDEX IF NOT EXISTS detection_category_idx
|
CREATE INDEX IF NOT EXISTS detection_category_idx
|
||||||
ON remote_cam.detection (category);
|
ON remote_cam.detection (category);
|
||||||
|
|
||||||
|
-- Einheitliches Label je Rohkategorie. detection.category bleibt die Rohbezeichnung der
|
||||||
|
-- Quelle (ICU-Gundi: "Deer-Like", Legacy-Klassifikator: "deer"); Auswertungen nutzen das Label.
|
||||||
|
CREATE TABLE IF NOT EXISTS remote_cam.category_label (
|
||||||
|
category text PRIMARY KEY,
|
||||||
|
label text NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO remote_cam.category_label (category, label) VALUES
|
||||||
|
('Deer-Like', 'deer'), ('deer', 'deer'),
|
||||||
|
('person', 'human'), ('human', 'human'),
|
||||||
|
('car', 'machine'), ('machine', 'machine'),
|
||||||
|
('Bird', 'bird'), ('bird', 'bird'),
|
||||||
|
('Fox', 'fox'), ('fox', 'fox'),
|
||||||
|
('Wild Boar', 'wild boar'), ('wild boar', 'wild boar'),
|
||||||
|
('racoon', 'raccoon'), ('raccoon', 'raccoon')
|
||||||
|
ON CONFLICT (category) DO NOTHING;
|
||||||
|
|
||||||
|
-- Ohne Eintrag in category_label gilt die kleingeschriebene Rohbezeichnung.
|
||||||
|
CREATE OR REPLACE VIEW remote_cam.detection_labeled AS
|
||||||
|
SELECT d.*, COALESCE(cl.label, lower(d.category)) AS label
|
||||||
|
FROM remote_cam.detection d
|
||||||
|
LEFT JOIN remote_cam.category_label cl ON cl.category = d.category;
|
||||||
|
|
||||||
|
-- Gundi-Bilder, die waehrend des Parallelbetriebs auch ueber den Legacy-Pfad kamen und
|
||||||
|
-- zugunsten des Legacy-Zwillings (traegt die KI-Ergebnisse) entfernt wurden. Haelt die
|
||||||
|
-- geloeschte image-Zeile samt Detections fuer Nachvollziehbarkeit und Rollback.
|
||||||
|
CREATE TABLE IF NOT EXISTS remote_cam.image_superseded (
|
||||||
|
image_uuid uuid PRIMARY KEY,
|
||||||
|
superseded_by uuid NOT NULL,
|
||||||
|
removed_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
image_row jsonb NOT NULL,
|
||||||
|
detections jsonb
|
||||||
|
);
|
||||||
|
|
||||||
-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine.
|
-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine.
|
||||||
CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox (
|
CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox (
|
||||||
object_id uuid PRIMARY KEY,
|
object_id uuid PRIMARY KEY,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt
|
-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt
|
||||||
-- ein image und ein camera_status; remote_cam.category wird zu detection.
|
-- ein image und ein camera_status; remote_cam.category (Score >= 0.5) wird zu detection.
|
||||||
--
|
--
|
||||||
-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py
|
-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py
|
||||||
-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen.
|
-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen.
|
||||||
@@ -82,12 +82,16 @@ WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
|
|||||||
ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING;
|
ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING;
|
||||||
|
|
||||||
-- step: detection
|
-- step: detection
|
||||||
|
-- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes:
|
||||||
|
-- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection).
|
||||||
INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source)
|
INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source)
|
||||||
SELECT i.image_uuid, c.category, c.score, c.version, 'entrance'
|
SELECT i.image_uuid, c.category, c.score, c.version, 'entrance'
|
||||||
FROM _mig_batch b
|
FROM _mig_batch b
|
||||||
JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid
|
JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid
|
||||||
JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid
|
JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid
|
||||||
WHERE c.category IS NOT NULL
|
WHERE c.category IS NOT NULL
|
||||||
|
AND c.category NOT IN ('', 'none')
|
||||||
|
AND c.score >= 0.5
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM remote_cam.detection d
|
SELECT 1 FROM remote_cam.detection d
|
||||||
WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance'
|
WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance'
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
-- Parallelbetrieb Legacy + Gundi: dasselbe Bild liegt doppelt in remote_cam.image.
|
||||||
|
-- Behalten wird der Legacy-Zwilling (source='entrance'): an seiner UUID haengen die
|
||||||
|
-- KI-Ergebnisse (ai_image/ai_object). Der Gundi-Zwilling wandert nach image_superseded.
|
||||||
|
--
|
||||||
|
-- Zwilling = gleiche imei, gleiche Dateigroesse, servertime hoechstens 10 min auseinander.
|
||||||
|
-- Gemessen am 2026-10-16: 4291 Paare, jedes Bild hat hoechstens einen Partner.
|
||||||
|
-- Nur Paare, deren Legacy-Seite schon migriert ist; Paare mit mehreren Kandidaten bleiben
|
||||||
|
-- unangetastet. Wiederholbar. tools/migrate_legacy.py --dedupe fuehrt die Schritte in EINER
|
||||||
|
-- Transaktion aus.
|
||||||
|
|
||||||
|
-- step: twins
|
||||||
|
CREATE TEMP TABLE _twins ON COMMIT DROP AS
|
||||||
|
WITH pairs AS (
|
||||||
|
SELECT g.image_uuid AS gundi_uuid, e.image_uuid AS legacy_uuid
|
||||||
|
FROM remote_cam.image g
|
||||||
|
JOIN remote_cam.image e
|
||||||
|
ON e.source = 'entrance'
|
||||||
|
AND e.imei = g.imei
|
||||||
|
AND e.size_bytes = g.size_bytes
|
||||||
|
AND e.servertime BETWEEN g.servertime - interval '10 minutes'
|
||||||
|
AND g.servertime + interval '10 minutes'
|
||||||
|
WHERE g.source = 'gundi'
|
||||||
|
)
|
||||||
|
SELECT gundi_uuid, legacy_uuid
|
||||||
|
FROM (SELECT p.*,
|
||||||
|
count(*) OVER (PARTITION BY p.gundi_uuid) AS n_gundi,
|
||||||
|
count(*) OVER (PARTITION BY p.legacy_uuid) AS n_legacy
|
||||||
|
FROM pairs p) x
|
||||||
|
WHERE n_gundi = 1 AND n_legacy = 1;
|
||||||
|
|
||||||
|
-- step: archive
|
||||||
|
INSERT INTO remote_cam.image_superseded (image_uuid, superseded_by, image_row, detections)
|
||||||
|
SELECT t.gundi_uuid, t.legacy_uuid, to_jsonb(i),
|
||||||
|
(SELECT jsonb_agg(to_jsonb(d)) FROM remote_cam.detection d WHERE d.image_uuid = t.gundi_uuid)
|
||||||
|
FROM _twins t
|
||||||
|
JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid
|
||||||
|
ON CONFLICT (image_uuid) DO NOTHING;
|
||||||
|
|
||||||
|
-- step: delete
|
||||||
|
-- detection haengt per ON DELETE CASCADE am image.
|
||||||
|
DELETE FROM remote_cam.image i
|
||||||
|
USING _twins t
|
||||||
|
WHERE i.image_uuid = t.gundi_uuid;
|
||||||
@@ -67,7 +67,7 @@ def fake_connect(cursor: FakeCursor):
|
|||||||
return lambda dsn: FakeConn(cursor)
|
return lambda dsn: FakeConn(cursor)
|
||||||
|
|
||||||
|
|
||||||
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
|
GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3)
|
||||||
|
|
||||||
|
|
||||||
class AttachImageTests(unittest.TestCase):
|
class AttachImageTests(unittest.TestCase):
|
||||||
@@ -89,7 +89,7 @@ class AttachImageTests(unittest.TestCase):
|
|||||||
s3 = mock.Mock()
|
s3 = mock.Mock()
|
||||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
||||||
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
|
s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||||
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
||||||
|
|
||||||
def test_already_received_is_duplicate_without_s3(self):
|
def test_already_received_is_duplicate_without_s3(self):
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
"""tools/migrate_legacy: Schrittdateien und --dedupe-Ablauf, ohne DB.
|
||||||
|
|
||||||
|
python -m unittest discover -s tests -v
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from tools import migrate_legacy as ml # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class FakeCursor:
|
||||||
|
def __init__(self, n_twins, deleted=None):
|
||||||
|
self.executed = []
|
||||||
|
self.n_twins = n_twins
|
||||||
|
self.deleted = n_twins if deleted is None else deleted
|
||||||
|
self.rowcount = 0
|
||||||
|
self._next = None
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def execute(self, sql, params=None):
|
||||||
|
self.executed.append(sql)
|
||||||
|
if "count(*) FROM _twins" in sql:
|
||||||
|
self._next = (self.n_twins,)
|
||||||
|
elif "min(i.servertime)" in sql:
|
||||||
|
self._next = (None, None)
|
||||||
|
elif sql.lstrip().startswith("INSERT INTO remote_cam.image_superseded"):
|
||||||
|
self.rowcount = self.n_twins
|
||||||
|
elif "DELETE FROM remote_cam.image" in sql:
|
||||||
|
self.rowcount = self.deleted
|
||||||
|
|
||||||
|
def fetchone(self):
|
||||||
|
return self._next
|
||||||
|
|
||||||
|
|
||||||
|
def fake_conn(cur):
|
||||||
|
conn = mock.Mock()
|
||||||
|
conn.cursor.return_value = cur
|
||||||
|
return conn
|
||||||
|
|
||||||
|
|
||||||
|
class StepFiles(unittest.TestCase):
|
||||||
|
def test_migration_steps(self):
|
||||||
|
steps = dict(ml.load_steps())
|
||||||
|
self.assertEqual(list(steps), ["image", "camera_status", "detection"])
|
||||||
|
det = steps["detection"]
|
||||||
|
self.assertIn("c.score >= 0.5", det)
|
||||||
|
self.assertIn("NOT IN ('', 'none')", det)
|
||||||
|
|
||||||
|
def test_dedupe_steps(self):
|
||||||
|
steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql"))
|
||||||
|
self.assertEqual(list(steps), ["twins", "archive", "delete"])
|
||||||
|
self.assertIn("e.size_bytes = g.size_bytes", steps["twins"])
|
||||||
|
self.assertIn("n_gundi = 1 AND n_legacy = 1", steps["twins"])
|
||||||
|
|
||||||
|
|
||||||
|
class Dedupe(unittest.TestCase):
|
||||||
|
def test_dry_run_changes_nothing(self):
|
||||||
|
cur = FakeCursor(n_twins=5)
|
||||||
|
conn = fake_conn(cur)
|
||||||
|
ml.dedupe(conn, dry_run=True)
|
||||||
|
self.assertFalse(any("image_superseded" in s or "DELETE" in s for s in cur.executed))
|
||||||
|
conn.rollback.assert_called_once()
|
||||||
|
conn.commit.assert_not_called()
|
||||||
|
|
||||||
|
def test_archives_then_deletes(self):
|
||||||
|
cur = FakeCursor(n_twins=5)
|
||||||
|
conn = fake_conn(cur)
|
||||||
|
ml.dedupe(conn, dry_run=False)
|
||||||
|
archive = next(i for i, s in enumerate(cur.executed) if "INSERT INTO remote_cam.image_superseded" in s)
|
||||||
|
delete = next(i for i, s in enumerate(cur.executed) if "DELETE FROM remote_cam.image" in s)
|
||||||
|
self.assertLess(archive, delete)
|
||||||
|
conn.commit.assert_called_once()
|
||||||
|
|
||||||
|
def test_count_mismatch_rolls_back(self):
|
||||||
|
cur = FakeCursor(n_twins=5, deleted=4)
|
||||||
|
conn = fake_conn(cur)
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
ml.dedupe(conn, dry_run=False)
|
||||||
|
conn.rollback.assert_called_once()
|
||||||
|
conn.commit.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+103
-5
@@ -4,8 +4,11 @@
|
|||||||
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
|
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
|
||||||
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
|
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
|
||||||
python -m tools.migrate_legacy --verify-only
|
python -m tools.migrate_legacy --verify-only
|
||||||
|
python -m tools.migrate_legacy --plan # nur lesen: was wuerde migriert/entfernt
|
||||||
|
python -m tools.migrate_legacy --dedupe --dry-run
|
||||||
|
python -m tools.migrate_legacy --dedupe # Gundi-Zwillinge entfernen (schema/003)
|
||||||
|
|
||||||
Wiederholbar. Jeder Batch ist eine eigene Transaktion.
|
Wiederholbar. Jeder Batch ist eine eigene Transaktion, --dedupe laeuft in einer.
|
||||||
"""
|
"""
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
@@ -20,8 +23,8 @@ SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
|
|||||||
SCHEMA_LOCK_KEY = 7_311_000_001
|
SCHEMA_LOCK_KEY = 7_311_000_001
|
||||||
|
|
||||||
|
|
||||||
def load_steps() -> list[tuple[str, str]]:
|
def load_steps(filename: str = "002_migrate_legacy.sql") -> list[tuple[str, str]]:
|
||||||
sql = (SCHEMA_DIR / "002_migrate_legacy.sql").read_text(encoding="utf-8")
|
sql = (SCHEMA_DIR / filename).read_text(encoding="utf-8")
|
||||||
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
|
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
|
||||||
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
|
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
|
||||||
|
|
||||||
@@ -74,6 +77,81 @@ def migrate(conn: psycopg.Connection, batch_size: int) -> None:
|
|||||||
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
|
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
# Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge).
|
||||||
|
PLAN_QUERIES = [
|
||||||
|
("image(entrance) neu",
|
||||||
|
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||||
|
"AND NOT EXISTS (SELECT 1 FROM remote_cam.image i WHERE i.image_uuid = m.metadata_uuid::uuid)"),
|
||||||
|
("camera_status(entrance) neu",
|
||||||
|
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||||
|
"AND NOT EXISTS (SELECT 1 FROM remote_cam.camera_status s WHERE s.source = 'entrance' "
|
||||||
|
"AND s.source_ref = m.metadata_uuid)"),
|
||||||
|
("detection(entrance) neu",
|
||||||
|
"SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid "
|
||||||
|
"WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||||
|
"AND c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND c.score >= 0.5 "
|
||||||
|
"AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid "
|
||||||
|
"AND d.source = 'entrance')"),
|
||||||
|
("Gundi-Zwillinge (nach Migration zu entfernen)",
|
||||||
|
"""
|
||||||
|
WITH pairs AS (
|
||||||
|
SELECT g.image_uuid AS gundi_uuid, m.metadata_uuid AS legacy_uuid
|
||||||
|
FROM remote_cam.image g
|
||||||
|
JOIN remote_cam.metadata m
|
||||||
|
ON m.imei = g.imei AND m."SizeInByte" = g.size_bytes
|
||||||
|
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes'
|
||||||
|
WHERE g.source = 'gundi')
|
||||||
|
SELECT count(*) FROM (
|
||||||
|
SELECT count(*) OVER (PARTITION BY gundi_uuid) AS n_g, count(*) OVER (PARTITION BY legacy_uuid) AS n_l
|
||||||
|
FROM pairs) x
|
||||||
|
WHERE n_g = 1 AND n_l = 1
|
||||||
|
"""),
|
||||||
|
("Gundi-Bilder ohne Zwilling (bleiben)",
|
||||||
|
"""
|
||||||
|
SELECT count(*) FROM remote_cam.image g WHERE g.source = 'gundi' AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM remote_cam.metadata m
|
||||||
|
WHERE m.imei = g.imei AND m."SizeInByte" = g.size_bytes
|
||||||
|
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes')
|
||||||
|
"""),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def plan(conn: psycopg.Connection) -> None:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("SET TRANSACTION READ ONLY")
|
||||||
|
for label, sql in PLAN_QUERIES:
|
||||||
|
cur.execute(sql)
|
||||||
|
print(f" {label}: {cur.fetchone()[0]}", flush=True)
|
||||||
|
conn.rollback()
|
||||||
|
|
||||||
|
|
||||||
|
def dedupe(conn: psycopg.Connection, dry_run: bool) -> None:
|
||||||
|
steps = dict(load_steps("003_dedupe_gundi_twins.sql"))
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(steps["twins"])
|
||||||
|
cur.execute("SELECT count(*) FROM _twins")
|
||||||
|
n = cur.fetchone()[0]
|
||||||
|
cur.execute(
|
||||||
|
"SELECT min(i.servertime), max(i.servertime) FROM _twins t "
|
||||||
|
"JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid"
|
||||||
|
)
|
||||||
|
lo, hi = cur.fetchone()
|
||||||
|
print(f" Gundi-Zwillinge: {n} (servertime {lo} .. {hi})", flush=True)
|
||||||
|
if dry_run:
|
||||||
|
conn.rollback()
|
||||||
|
print(" Trockenlauf: nichts geaendert.", flush=True)
|
||||||
|
return
|
||||||
|
cur.execute(steps["archive"])
|
||||||
|
archived = cur.rowcount
|
||||||
|
cur.execute(steps["delete"])
|
||||||
|
deleted = cur.rowcount
|
||||||
|
if deleted != n:
|
||||||
|
conn.rollback()
|
||||||
|
raise RuntimeError(f"geloescht={deleted} != Zwillinge={n}; zurueckgerollt")
|
||||||
|
conn.commit()
|
||||||
|
print(f" archiviert={archived} geloescht={deleted}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
def verify(conn: psycopg.Connection) -> bool:
|
def verify(conn: psycopg.Connection) -> bool:
|
||||||
checks = [
|
checks = [
|
||||||
("metadata == image(entrance)",
|
("metadata == image(entrance)",
|
||||||
@@ -82,9 +160,16 @@ def verify(conn: psycopg.Connection) -> bool:
|
|||||||
("metadata == camera_status(entrance)",
|
("metadata == camera_status(entrance)",
|
||||||
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
||||||
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
|
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
|
||||||
("category == detection(entrance)",
|
("category (Score >= 0.5, ohne ''/none) == detection(entrance)",
|
||||||
"SELECT (SELECT count(*) FROM remote_cam.category), "
|
"SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m "
|
||||||
|
"ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||||
|
"AND c.category NOT IN ('', 'none') AND c.score >= 0.5), "
|
||||||
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
|
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
|
||||||
|
("keine Gundi-Bilder mit Legacy-Zwilling",
|
||||||
|
"SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e "
|
||||||
|
"ON e.source = 'entrance' AND e.imei = g.imei AND e.size_bytes = g.size_bytes "
|
||||||
|
"AND e.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes' "
|
||||||
|
"WHERE g.source = 'gundi'"),
|
||||||
("metadata ohne ocr_ts == image FAIL",
|
("metadata ohne ocr_ts == image FAIL",
|
||||||
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
|
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
|
||||||
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
|
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
|
||||||
@@ -124,12 +209,25 @@ def main() -> int:
|
|||||||
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
|
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
|
||||||
ap.add_argument("--batch", type=int, default=20000)
|
ap.add_argument("--batch", type=int, default=20000)
|
||||||
ap.add_argument("--verify-only", action="store_true")
|
ap.add_argument("--verify-only", action="store_true")
|
||||||
|
ap.add_argument("--plan", action="store_true", help="nur lesen: Vorschau der Mengen")
|
||||||
|
ap.add_argument("--dedupe", action="store_true", help="Gundi-Zwillinge entfernen (schema/003)")
|
||||||
|
ap.add_argument("--dry-run", action="store_true", help="mit --dedupe: nur zaehlen")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
if not args.dsn:
|
if not args.dsn:
|
||||||
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
|
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
|
|
||||||
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
|
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
|
||||||
|
if args.plan:
|
||||||
|
print("Vorschau (nur lesend):", flush=True)
|
||||||
|
plan(conn)
|
||||||
|
return 0
|
||||||
|
if args.dedupe:
|
||||||
|
if not args.dry_run:
|
||||||
|
apply_schema(conn)
|
||||||
|
print("Gundi-Zwillinge:", flush=True)
|
||||||
|
dedupe(conn, args.dry_run)
|
||||||
|
return 0
|
||||||
if not args.verify_only:
|
if not args.verify_only:
|
||||||
apply_schema(conn)
|
apply_schema(conn)
|
||||||
print("Schema angewendet. Migriere ...", flush=True)
|
print("Schema angewendet. Migriere ...", flush=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user