Prepare legacy migration for the portal cutover

- detection from legacy category only for scores >= 0.5, without ''/none
- category_label + view detection_labeled unify ICU and legacy labels
- schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images
  (same imei, same size, <= 10 min), archived in image_superseded
- migrate_legacy --plan: read-only preview of the migration
- Gundi S3 prefix icu/gundi-test/ -> icu/gundi/

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-16 21:01:17 +02:00
co-authored by Claude Opus 5.5
parent e7e5b6fb87
commit ed105fbff8
9 changed files with 307 additions and 22 deletions
+2 -2
View File
@@ -67,7 +67,7 @@ def fake_connect(cursor: FakeCursor):
return lambda dsn: FakeConn(cursor)
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3)
class AttachImageTests(unittest.TestCase):
@@ -89,7 +89,7 @@ class AttachImageTests(unittest.TestCase):
s3 = mock.Mock()
with mock.patch.object(gp, "connect", fake_connect(cur)):
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg")
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
def test_already_received_is_duplicate_without_s3(self):
+93
View File
@@ -0,0 +1,93 @@
"""tools/migrate_legacy: Schrittdateien und --dedupe-Ablauf, ohne DB.
python -m unittest discover -s tests -v
"""
import os
import sys
import unittest
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from tools import migrate_legacy as ml # noqa: E402
class FakeCursor:
def __init__(self, n_twins, deleted=None):
self.executed = []
self.n_twins = n_twins
self.deleted = n_twins if deleted is None else deleted
self.rowcount = 0
self._next = None
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def execute(self, sql, params=None):
self.executed.append(sql)
if "count(*) FROM _twins" in sql:
self._next = (self.n_twins,)
elif "min(i.servertime)" in sql:
self._next = (None, None)
elif sql.lstrip().startswith("INSERT INTO remote_cam.image_superseded"):
self.rowcount = self.n_twins
elif "DELETE FROM remote_cam.image" in sql:
self.rowcount = self.deleted
def fetchone(self):
return self._next
def fake_conn(cur):
conn = mock.Mock()
conn.cursor.return_value = cur
return conn
class StepFiles(unittest.TestCase):
def test_migration_steps(self):
steps = dict(ml.load_steps())
self.assertEqual(list(steps), ["image", "camera_status", "detection"])
det = steps["detection"]
self.assertIn("c.score >= 0.5", det)
self.assertIn("NOT IN ('', 'none')", det)
def test_dedupe_steps(self):
steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql"))
self.assertEqual(list(steps), ["twins", "archive", "delete"])
self.assertIn("e.size_bytes = g.size_bytes", steps["twins"])
self.assertIn("n_gundi = 1 AND n_legacy = 1", steps["twins"])
class Dedupe(unittest.TestCase):
def test_dry_run_changes_nothing(self):
cur = FakeCursor(n_twins=5)
conn = fake_conn(cur)
ml.dedupe(conn, dry_run=True)
self.assertFalse(any("image_superseded" in s or "DELETE" in s for s in cur.executed))
conn.rollback.assert_called_once()
conn.commit.assert_not_called()
def test_archives_then_deletes(self):
cur = FakeCursor(n_twins=5)
conn = fake_conn(cur)
ml.dedupe(conn, dry_run=False)
archive = next(i for i, s in enumerate(cur.executed) if "INSERT INTO remote_cam.image_superseded" in s)
delete = next(i for i, s in enumerate(cur.executed) if "DELETE FROM remote_cam.image" in s)
self.assertLess(archive, delete)
conn.commit.assert_called_once()
def test_count_mismatch_rolls_back(self):
cur = FakeCursor(n_twins=5, deleted=4)
conn = fake_conn(cur)
with self.assertRaises(RuntimeError):
ml.dedupe(conn, dry_run=False)
conn.rollback.assert_called_once()
conn.commit.assert_not_called()
if __name__ == "__main__":
unittest.main()