diff --git a/schema/002_migrate_legacy.sql b/schema/002_migrate_legacy.sql index a95df96..87816d8 100644 --- a/schema/002_migrate_legacy.sql +++ b/schema/002_migrate_legacy.sql @@ -84,6 +84,9 @@ ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING; -- step: detection -- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes: -- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection). +-- Ausnahme Datenschutz: war 'human' oder 'machine' die Top-Kategorie des Bildes, wird sie auch +-- unter 0.5 uebernommen -- das Portal sperrt solche Bilder fuer Nicht-Admins (wie bisher ueber +-- die Top-Kategorie). Tiebreaker wie die fruehere image_top_category: Score, dann Name. INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source) SELECT i.image_uuid, c.category, c.score, c.version, 'entrance' FROM _mig_batch b @@ -91,7 +94,12 @@ JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid WHERE c.category IS NOT NULL AND c.category NOT IN ('', 'none') - AND c.score >= 0.5 + AND (c.score >= 0.5 + OR (c.category IN ('human', 'machine') + AND NOT EXISTS ( + SELECT 1 FROM remote_cam.category c2 + WHERE c2.image_uuid = c.image_uuid + AND (c2.score > c.score OR (c2.score = c.score AND c2.category < c.category))))) AND NOT EXISTS ( SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance' diff --git a/tests/test_migrate_legacy.py b/tests/test_migrate_legacy.py index 667e0f4..8f7896d 100644 --- a/tests/test_migrate_legacy.py +++ b/tests/test_migrate_legacy.py @@ -54,6 +54,8 @@ class StepFiles(unittest.TestCase): det = steps["detection"] self.assertIn("c.score >= 0.5", det) self.assertIn("NOT IN ('', 'none')", det) + self.assertIn("c.category IN ('human', 'machine')", det) + self.assertIn("c.category IN ('human', 'machine')", ml.DETECTION_FILTER) def test_dedupe_steps(self): steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql")) diff --git a/tools/migrate_legacy.py b/tools/migrate_legacy.py index c88dfad..a2ed5a9 100644 --- a/tools/migrate_legacy.py +++ b/tools/migrate_legacy.py @@ -77,6 +77,14 @@ def migrate(conn: psycopg.Connection, batch_size: int) -> None: print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True) +# Welche Zeilen aus remote_cam.category (Alias c) zu detection werden -- wie in 002. +DETECTION_FILTER = ( + "c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND (c.score >= 0.5 " + "OR (c.category IN ('human', 'machine') AND NOT EXISTS (SELECT 1 FROM remote_cam.category c2 " + "WHERE c2.image_uuid = c.image_uuid " + "AND (c2.score > c.score OR (c2.score = c.score AND c2.category < c.category)))))" +) + # Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge). PLAN_QUERIES = [ ("image(entrance) neu", @@ -88,8 +96,7 @@ PLAN_QUERIES = [ "AND s.source_ref = m.metadata_uuid)"), ("detection(entrance) neu", "SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid " - "WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " - "AND c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND c.score >= 0.5 " + "WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL AND " + DETECTION_FILTER + " " "AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid " "AND d.source = 'entrance')"), ("Gundi-Zwillinge (nach Migration zu entfernen)", @@ -160,10 +167,10 @@ def verify(conn: psycopg.Connection) -> bool: ("metadata == camera_status(entrance)", "SELECT (SELECT count(*) FROM remote_cam.metadata), " "(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"), - ("category (Score >= 0.5, ohne ''/none) == detection(entrance)", + ("category (Score >= 0.5 bzw. Top human/machine, ohne ''/none) == detection(entrance)", "SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m " "ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " - "AND c.category NOT IN ('', 'none') AND c.score >= 0.5), " + "AND " + DETECTION_FILTER + "), " "(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"), ("keine Gundi-Bilder mit Legacy-Zwilling", "SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e "