- detection from legacy category only for scores >= 0.5, without ''/none - category_label + view detection_labeled unify ICU and legacy labels - schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images (same imei, same size, <= 10 min), archived in image_superseded - migrate_legacy --plan: read-only preview of the migration - Gundi S3 prefix icu/gundi-test/ -> icu/gundi/ Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
3.0 KiB
Python
94 lines
3.0 KiB
Python
"""tools/migrate_legacy: Schrittdateien und --dedupe-Ablauf, ohne DB.
|
|
|
|
python -m unittest discover -s tests -v
|
|
"""
|
|
import os
|
|
import sys
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from tools import migrate_legacy as ml # noqa: E402
|
|
|
|
|
|
class FakeCursor:
|
|
def __init__(self, n_twins, deleted=None):
|
|
self.executed = []
|
|
self.n_twins = n_twins
|
|
self.deleted = n_twins if deleted is None else deleted
|
|
self.rowcount = 0
|
|
self._next = None
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def execute(self, sql, params=None):
|
|
self.executed.append(sql)
|
|
if "count(*) FROM _twins" in sql:
|
|
self._next = (self.n_twins,)
|
|
elif "min(i.servertime)" in sql:
|
|
self._next = (None, None)
|
|
elif sql.lstrip().startswith("INSERT INTO remote_cam.image_superseded"):
|
|
self.rowcount = self.n_twins
|
|
elif "DELETE FROM remote_cam.image" in sql:
|
|
self.rowcount = self.deleted
|
|
|
|
def fetchone(self):
|
|
return self._next
|
|
|
|
|
|
def fake_conn(cur):
|
|
conn = mock.Mock()
|
|
conn.cursor.return_value = cur
|
|
return conn
|
|
|
|
|
|
class StepFiles(unittest.TestCase):
|
|
def test_migration_steps(self):
|
|
steps = dict(ml.load_steps())
|
|
self.assertEqual(list(steps), ["image", "camera_status", "detection"])
|
|
det = steps["detection"]
|
|
self.assertIn("c.score >= 0.5", det)
|
|
self.assertIn("NOT IN ('', 'none')", det)
|
|
|
|
def test_dedupe_steps(self):
|
|
steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql"))
|
|
self.assertEqual(list(steps), ["twins", "archive", "delete"])
|
|
self.assertIn("e.size_bytes = g.size_bytes", steps["twins"])
|
|
self.assertIn("n_gundi = 1 AND n_legacy = 1", steps["twins"])
|
|
|
|
|
|
class Dedupe(unittest.TestCase):
|
|
def test_dry_run_changes_nothing(self):
|
|
cur = FakeCursor(n_twins=5)
|
|
conn = fake_conn(cur)
|
|
ml.dedupe(conn, dry_run=True)
|
|
self.assertFalse(any("image_superseded" in s or "DELETE" in s for s in cur.executed))
|
|
conn.rollback.assert_called_once()
|
|
conn.commit.assert_not_called()
|
|
|
|
def test_archives_then_deletes(self):
|
|
cur = FakeCursor(n_twins=5)
|
|
conn = fake_conn(cur)
|
|
ml.dedupe(conn, dry_run=False)
|
|
archive = next(i for i, s in enumerate(cur.executed) if "INSERT INTO remote_cam.image_superseded" in s)
|
|
delete = next(i for i, s in enumerate(cur.executed) if "DELETE FROM remote_cam.image" in s)
|
|
self.assertLess(archive, delete)
|
|
conn.commit.assert_called_once()
|
|
|
|
def test_count_mismatch_rolls_back(self):
|
|
cur = FakeCursor(n_twins=5, deleted=4)
|
|
conn = fake_conn(cur)
|
|
with self.assertRaises(RuntimeError):
|
|
ml.dedupe(conn, dry_run=False)
|
|
conn.rollback.assert_called_once()
|
|
conn.commit.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|