Prepare legacy migration for the portal cutover
- detection from legacy category only for scores >= 0.5, without ''/none - category_label + view detection_labeled unify ICU and legacy labels - schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images (same imei, same size, <= 10 min), archived in image_superseded - migrate_legacy --plan: read-only preview of the migration - Gundi S3 prefix icu/gundi-test/ -> icu/gundi/ Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -67,7 +67,7 @@ def fake_connect(cursor: FakeCursor):
|
||||
return lambda dsn: FakeConn(cursor)
|
||||
|
||||
|
||||
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
|
||||
GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3)
|
||||
|
||||
|
||||
class AttachImageTests(unittest.TestCase):
|
||||
@@ -89,7 +89,7 @@ class AttachImageTests(unittest.TestCase):
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
||||
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||
s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
||||
|
||||
def test_already_received_is_duplicate_without_s3(self):
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""tools/migrate_legacy: Schrittdateien und --dedupe-Ablauf, ohne DB.
|
||||
|
||||
python -m unittest discover -s tests -v
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from tools import migrate_legacy as ml # noqa: E402
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, n_twins, deleted=None):
|
||||
self.executed = []
|
||||
self.n_twins = n_twins
|
||||
self.deleted = n_twins if deleted is None else deleted
|
||||
self.rowcount = 0
|
||||
self._next = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.executed.append(sql)
|
||||
if "count(*) FROM _twins" in sql:
|
||||
self._next = (self.n_twins,)
|
||||
elif "min(i.servertime)" in sql:
|
||||
self._next = (None, None)
|
||||
elif sql.lstrip().startswith("INSERT INTO remote_cam.image_superseded"):
|
||||
self.rowcount = self.n_twins
|
||||
elif "DELETE FROM remote_cam.image" in sql:
|
||||
self.rowcount = self.deleted
|
||||
|
||||
def fetchone(self):
|
||||
return self._next
|
||||
|
||||
|
||||
def fake_conn(cur):
|
||||
conn = mock.Mock()
|
||||
conn.cursor.return_value = cur
|
||||
return conn
|
||||
|
||||
|
||||
class StepFiles(unittest.TestCase):
|
||||
def test_migration_steps(self):
|
||||
steps = dict(ml.load_steps())
|
||||
self.assertEqual(list(steps), ["image", "camera_status", "detection"])
|
||||
det = steps["detection"]
|
||||
self.assertIn("c.score >= 0.5", det)
|
||||
self.assertIn("NOT IN ('', 'none')", det)
|
||||
|
||||
def test_dedupe_steps(self):
|
||||
steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql"))
|
||||
self.assertEqual(list(steps), ["twins", "archive", "delete"])
|
||||
self.assertIn("e.size_bytes = g.size_bytes", steps["twins"])
|
||||
self.assertIn("n_gundi = 1 AND n_legacy = 1", steps["twins"])
|
||||
|
||||
|
||||
class Dedupe(unittest.TestCase):
|
||||
def test_dry_run_changes_nothing(self):
|
||||
cur = FakeCursor(n_twins=5)
|
||||
conn = fake_conn(cur)
|
||||
ml.dedupe(conn, dry_run=True)
|
||||
self.assertFalse(any("image_superseded" in s or "DELETE" in s for s in cur.executed))
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
def test_archives_then_deletes(self):
|
||||
cur = FakeCursor(n_twins=5)
|
||||
conn = fake_conn(cur)
|
||||
ml.dedupe(conn, dry_run=False)
|
||||
archive = next(i for i, s in enumerate(cur.executed) if "INSERT INTO remote_cam.image_superseded" in s)
|
||||
delete = next(i for i, s in enumerate(cur.executed) if "DELETE FROM remote_cam.image" in s)
|
||||
self.assertLess(archive, delete)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
def test_count_mismatch_rolls_back(self):
|
||||
cur = FakeCursor(n_twins=5, deleted=4)
|
||||
conn = fake_conn(cur)
|
||||
with self.assertRaises(RuntimeError):
|
||||
ml.dedupe(conn, dry_run=False)
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user