From 39b22f01248457eab804b67dbbd0c944cae357f2 Mon Sep 17 00:00:00 2001 From: Dominik Dachs Date: Fri, 16 Oct 2026 22:15:12 +0200 Subject: [PATCH] Keep top human/machine legacy categories below 0.5 for privacy blocking The portal blocks images whose top ICU category is human or machine for non-admins. With the 0.5 threshold, 103 such images would have lost their detection and become visible. Those categories are now migrated when they were the image's top category, regardless of score. Co-Authored-By: Claude Opus 5.5 --- schema/002_migrate_legacy.sql | 10 +++++++++- tests/test_migrate_legacy.py | 2 ++ tools/migrate_legacy.py | 15 +++++++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/schema/002_migrate_legacy.sql b/schema/002_migrate_legacy.sql index a95df96..87816d8 100644 --- a/schema/002_migrate_legacy.sql +++ b/schema/002_migrate_legacy.sql @@ -84,6 +84,9 @@ ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING; -- step: detection -- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes: -- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection). +-- Ausnahme Datenschutz: war 'human' oder 'machine' die Top-Kategorie des Bildes, wird sie auch +-- unter 0.5 uebernommen -- das Portal sperrt solche Bilder fuer Nicht-Admins (wie bisher ueber +-- die Top-Kategorie). Tiebreaker wie die fruehere image_top_category: Score, dann Name. INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source) SELECT i.image_uuid, c.category, c.score, c.version, 'entrance' FROM _mig_batch b @@ -91,7 +94,12 @@ JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid WHERE c.category IS NOT NULL AND c.category NOT IN ('', 'none') - AND c.score >= 0.5 + AND (c.score >= 0.5 + OR (c.category IN ('human', 'machine') + AND NOT EXISTS ( + SELECT 1 FROM remote_cam.category c2 + WHERE c2.image_uuid = c.image_uuid + AND (c2.score > c.score OR (c2.score = c.score AND c2.category < c.category))))) AND NOT EXISTS ( SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance' diff --git a/tests/test_migrate_legacy.py b/tests/test_migrate_legacy.py index 667e0f4..8f7896d 100644 --- a/tests/test_migrate_legacy.py +++ b/tests/test_migrate_legacy.py @@ -54,6 +54,8 @@ class StepFiles(unittest.TestCase): det = steps["detection"] self.assertIn("c.score >= 0.5", det) self.assertIn("NOT IN ('', 'none')", det) + self.assertIn("c.category IN ('human', 'machine')", det) + self.assertIn("c.category IN ('human', 'machine')", ml.DETECTION_FILTER) def test_dedupe_steps(self): steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql")) diff --git a/tools/migrate_legacy.py b/tools/migrate_legacy.py index c88dfad..a2ed5a9 100644 --- a/tools/migrate_legacy.py +++ b/tools/migrate_legacy.py @@ -77,6 +77,14 @@ def migrate(conn: psycopg.Connection, batch_size: int) -> None: print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True) +# Welche Zeilen aus remote_cam.category (Alias c) zu detection werden -- wie in 002. +DETECTION_FILTER = ( + "c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND (c.score >= 0.5 " + "OR (c.category IN ('human', 'machine') AND NOT EXISTS (SELECT 1 FROM remote_cam.category c2 " + "WHERE c2.image_uuid = c.image_uuid " + "AND (c2.score > c.score OR (c2.score = c.score AND c2.category < c.category)))))" +) + # Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge). PLAN_QUERIES = [ ("image(entrance) neu", @@ -88,8 +96,7 @@ PLAN_QUERIES = [ "AND s.source_ref = m.metadata_uuid)"), ("detection(entrance) neu", "SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid " - "WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " - "AND c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND c.score >= 0.5 " + "WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL AND " + DETECTION_FILTER + " " "AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid " "AND d.source = 'entrance')"), ("Gundi-Zwillinge (nach Migration zu entfernen)", @@ -160,10 +167,10 @@ def verify(conn: psycopg.Connection) -> bool: ("metadata == camera_status(entrance)", "SELECT (SELECT count(*) FROM remote_cam.metadata), " "(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"), - ("category (Score >= 0.5, ohne ''/none) == detection(entrance)", + ("category (Score >= 0.5 bzw. Top human/machine, ohne ''/none) == detection(entrance)", "SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m " "ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL " - "AND c.category NOT IN ('', 'none') AND c.score >= 0.5), " + "AND " + DETECTION_FILTER + "), " "(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"), ("keine Gundi-Bilder mit Legacy-Zwilling", "SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e "