Prepare legacy migration for the portal cutover

- detection from legacy category only for scores >= 0.5, without ''/none
- category_label + view detection_labeled unify ICU and legacy labels
- schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images
  (same imei, same size, <= 10 min), archived in image_superseded
- migrate_legacy --plan: read-only preview of the migration
- Gundi S3 prefix icu/gundi-test/ -> icu/gundi/

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-16 21:01:17 +02:00
co-authored by Claude Opus 5.5
parent e7e5b6fb87
commit ed105fbff8
9 changed files with 307 additions and 22 deletions
+34
View File
@@ -98,6 +98,40 @@ CREATE INDEX IF NOT EXISTS detection_image_confidence_idx
CREATE INDEX IF NOT EXISTS detection_category_idx
ON remote_cam.detection (category);
-- Einheitliches Label je Rohkategorie. detection.category bleibt die Rohbezeichnung der
-- Quelle (ICU-Gundi: "Deer-Like", Legacy-Klassifikator: "deer"); Auswertungen nutzen das Label.
CREATE TABLE IF NOT EXISTS remote_cam.category_label (
category text PRIMARY KEY,
label text NOT NULL
);
INSERT INTO remote_cam.category_label (category, label) VALUES
('Deer-Like', 'deer'), ('deer', 'deer'),
('person', 'human'), ('human', 'human'),
('car', 'machine'), ('machine', 'machine'),
('Bird', 'bird'), ('bird', 'bird'),
('Fox', 'fox'), ('fox', 'fox'),
('Wild Boar', 'wild boar'), ('wild boar', 'wild boar'),
('racoon', 'raccoon'), ('raccoon', 'raccoon')
ON CONFLICT (category) DO NOTHING;
-- Ohne Eintrag in category_label gilt die kleingeschriebene Rohbezeichnung.
CREATE OR REPLACE VIEW remote_cam.detection_labeled AS
SELECT d.*, COALESCE(cl.label, lower(d.category)) AS label
FROM remote_cam.detection d
LEFT JOIN remote_cam.category_label cl ON cl.category = d.category;
-- Gundi-Bilder, die waehrend des Parallelbetriebs auch ueber den Legacy-Pfad kamen und
-- zugunsten des Legacy-Zwillings (traegt die KI-Ergebnisse) entfernt wurden. Haelt die
-- geloeschte image-Zeile samt Detections fuer Nachvollziehbarkeit und Rollback.
CREATE TABLE IF NOT EXISTS remote_cam.image_superseded (
image_uuid uuid PRIMARY KEY,
superseded_by uuid NOT NULL,
removed_at timestamptz NOT NULL DEFAULT now(),
image_row jsonb NOT NULL,
detections jsonb
);
-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine.
CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox (
object_id uuid PRIMARY KEY,
+5 -1
View File
@@ -1,5 +1,5 @@
-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt
-- ein image und ein camera_status; remote_cam.category wird zu detection.
-- ein image und ein camera_status; remote_cam.category (Score >= 0.5) wird zu detection.
--
-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py
-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen.
@@ -82,12 +82,16 @@ WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING;
-- step: detection
-- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes:
-- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection).
INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source)
SELECT i.image_uuid, c.category, c.score, c.version, 'entrance'
FROM _mig_batch b
JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid
JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid
WHERE c.category IS NOT NULL
AND c.category NOT IN ('', 'none')
AND c.score >= 0.5
AND NOT EXISTS (
SELECT 1 FROM remote_cam.detection d
WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance'
+43
View File
@@ -0,0 +1,43 @@
-- Parallelbetrieb Legacy + Gundi: dasselbe Bild liegt doppelt in remote_cam.image.
-- Behalten wird der Legacy-Zwilling (source='entrance'): an seiner UUID haengen die
-- KI-Ergebnisse (ai_image/ai_object). Der Gundi-Zwilling wandert nach image_superseded.
--
-- Zwilling = gleiche imei, gleiche Dateigroesse, servertime hoechstens 10 min auseinander.
-- Gemessen am 2026-10-16: 4291 Paare, jedes Bild hat hoechstens einen Partner.
-- Nur Paare, deren Legacy-Seite schon migriert ist; Paare mit mehreren Kandidaten bleiben
-- unangetastet. Wiederholbar. tools/migrate_legacy.py --dedupe fuehrt die Schritte in EINER
-- Transaktion aus.
-- step: twins
CREATE TEMP TABLE _twins ON COMMIT DROP AS
WITH pairs AS (
SELECT g.image_uuid AS gundi_uuid, e.image_uuid AS legacy_uuid
FROM remote_cam.image g
JOIN remote_cam.image e
ON e.source = 'entrance'
AND e.imei = g.imei
AND e.size_bytes = g.size_bytes
AND e.servertime BETWEEN g.servertime - interval '10 minutes'
AND g.servertime + interval '10 minutes'
WHERE g.source = 'gundi'
)
SELECT gundi_uuid, legacy_uuid
FROM (SELECT p.*,
count(*) OVER (PARTITION BY p.gundi_uuid) AS n_gundi,
count(*) OVER (PARTITION BY p.legacy_uuid) AS n_legacy
FROM pairs p) x
WHERE n_gundi = 1 AND n_legacy = 1;
-- step: archive
INSERT INTO remote_cam.image_superseded (image_uuid, superseded_by, image_row, detections)
SELECT t.gundi_uuid, t.legacy_uuid, to_jsonb(i),
(SELECT jsonb_agg(to_jsonb(d)) FROM remote_cam.detection d WHERE d.image_uuid = t.gundi_uuid)
FROM _twins t
JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid
ON CONFLICT (image_uuid) DO NOTHING;
-- step: delete
-- detection haengt per ON DELETE CASCADE am image.
DELETE FROM remote_cam.image i
USING _twins t
WHERE i.image_uuid = t.gundi_uuid;