Prepare legacy migration for the portal cutover
- detection from legacy category only for scores >= 0.5, without ''/none - category_label + view detection_labeled unify ICU and legacy labels - schema/003 + migrate_legacy --dedupe removes Gundi twins of legacy images (same imei, same size, <= 10 min), archived in image_superseded - migrate_legacy --plan: read-only preview of the migration - Gundi S3 prefix icu/gundi-test/ -> icu/gundi/ Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -98,6 +98,40 @@ CREATE INDEX IF NOT EXISTS detection_image_confidence_idx
|
||||
CREATE INDEX IF NOT EXISTS detection_category_idx
|
||||
ON remote_cam.detection (category);
|
||||
|
||||
-- Einheitliches Label je Rohkategorie. detection.category bleibt die Rohbezeichnung der
|
||||
-- Quelle (ICU-Gundi: "Deer-Like", Legacy-Klassifikator: "deer"); Auswertungen nutzen das Label.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.category_label (
|
||||
category text PRIMARY KEY,
|
||||
label text NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO remote_cam.category_label (category, label) VALUES
|
||||
('Deer-Like', 'deer'), ('deer', 'deer'),
|
||||
('person', 'human'), ('human', 'human'),
|
||||
('car', 'machine'), ('machine', 'machine'),
|
||||
('Bird', 'bird'), ('bird', 'bird'),
|
||||
('Fox', 'fox'), ('fox', 'fox'),
|
||||
('Wild Boar', 'wild boar'), ('wild boar', 'wild boar'),
|
||||
('racoon', 'raccoon'), ('raccoon', 'raccoon')
|
||||
ON CONFLICT (category) DO NOTHING;
|
||||
|
||||
-- Ohne Eintrag in category_label gilt die kleingeschriebene Rohbezeichnung.
|
||||
CREATE OR REPLACE VIEW remote_cam.detection_labeled AS
|
||||
SELECT d.*, COALESCE(cl.label, lower(d.category)) AS label
|
||||
FROM remote_cam.detection d
|
||||
LEFT JOIN remote_cam.category_label cl ON cl.category = d.category;
|
||||
|
||||
-- Gundi-Bilder, die waehrend des Parallelbetriebs auch ueber den Legacy-Pfad kamen und
|
||||
-- zugunsten des Legacy-Zwillings (traegt die KI-Ergebnisse) entfernt wurden. Haelt die
|
||||
-- geloeschte image-Zeile samt Detections fuer Nachvollziehbarkeit und Rollback.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.image_superseded (
|
||||
image_uuid uuid PRIMARY KEY,
|
||||
superseded_by uuid NOT NULL,
|
||||
removed_at timestamptz NOT NULL DEFAULT now(),
|
||||
image_row jsonb NOT NULL,
|
||||
detections jsonb
|
||||
);
|
||||
|
||||
-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox (
|
||||
object_id uuid PRIMARY KEY,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt
|
||||
-- ein image und ein camera_status; remote_cam.category wird zu detection.
|
||||
-- ein image und ein camera_status; remote_cam.category (Score >= 0.5) wird zu detection.
|
||||
--
|
||||
-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py
|
||||
-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen.
|
||||
@@ -82,12 +82,16 @@ WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
|
||||
ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING;
|
||||
|
||||
-- step: detection
|
||||
-- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes:
|
||||
-- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection).
|
||||
INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source)
|
||||
SELECT i.image_uuid, c.category, c.score, c.version, 'entrance'
|
||||
FROM _mig_batch b
|
||||
JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid
|
||||
JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid
|
||||
WHERE c.category IS NOT NULL
|
||||
AND c.category NOT IN ('', 'none')
|
||||
AND c.score >= 0.5
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM remote_cam.detection d
|
||||
WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance'
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Parallelbetrieb Legacy + Gundi: dasselbe Bild liegt doppelt in remote_cam.image.
|
||||
-- Behalten wird der Legacy-Zwilling (source='entrance'): an seiner UUID haengen die
|
||||
-- KI-Ergebnisse (ai_image/ai_object). Der Gundi-Zwilling wandert nach image_superseded.
|
||||
--
|
||||
-- Zwilling = gleiche imei, gleiche Dateigroesse, servertime hoechstens 10 min auseinander.
|
||||
-- Gemessen am 2026-10-16: 4291 Paare, jedes Bild hat hoechstens einen Partner.
|
||||
-- Nur Paare, deren Legacy-Seite schon migriert ist; Paare mit mehreren Kandidaten bleiben
|
||||
-- unangetastet. Wiederholbar. tools/migrate_legacy.py --dedupe fuehrt die Schritte in EINER
|
||||
-- Transaktion aus.
|
||||
|
||||
-- step: twins
|
||||
CREATE TEMP TABLE _twins ON COMMIT DROP AS
|
||||
WITH pairs AS (
|
||||
SELECT g.image_uuid AS gundi_uuid, e.image_uuid AS legacy_uuid
|
||||
FROM remote_cam.image g
|
||||
JOIN remote_cam.image e
|
||||
ON e.source = 'entrance'
|
||||
AND e.imei = g.imei
|
||||
AND e.size_bytes = g.size_bytes
|
||||
AND e.servertime BETWEEN g.servertime - interval '10 minutes'
|
||||
AND g.servertime + interval '10 minutes'
|
||||
WHERE g.source = 'gundi'
|
||||
)
|
||||
SELECT gundi_uuid, legacy_uuid
|
||||
FROM (SELECT p.*,
|
||||
count(*) OVER (PARTITION BY p.gundi_uuid) AS n_gundi,
|
||||
count(*) OVER (PARTITION BY p.legacy_uuid) AS n_legacy
|
||||
FROM pairs p) x
|
||||
WHERE n_gundi = 1 AND n_legacy = 1;
|
||||
|
||||
-- step: archive
|
||||
INSERT INTO remote_cam.image_superseded (image_uuid, superseded_by, image_row, detections)
|
||||
SELECT t.gundi_uuid, t.legacy_uuid, to_jsonb(i),
|
||||
(SELECT jsonb_agg(to_jsonb(d)) FROM remote_cam.detection d WHERE d.image_uuid = t.gundi_uuid)
|
||||
FROM _twins t
|
||||
JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid
|
||||
ON CONFLICT (image_uuid) DO NOTHING;
|
||||
|
||||
-- step: delete
|
||||
-- detection haengt per ON DELETE CASCADE am image.
|
||||
DELETE FROM remote_cam.image i
|
||||
USING _twins t
|
||||
WHERE i.image_uuid = t.gundi_uuid;
|
||||
Reference in New Issue
Block a user