Add Gundi v2 push receiver for ICU and normalized data model

ICU now pushes camera events via a Gundi-v2-compatible API instead of
dropping file triplets into S3. The new gundi-receiver service accepts
them directly, so images are in the system within seconds instead of
waiting out the 90s age gate + poll interval. OCR of the burnt-in camera
timestamp stays mandatory (ICU only sends server times; OCR detects
drifting camera clocks) but now runs asynchronously after ingest.

The push path writes a normalized model that separates what the legacy
metadata table conflates: camera_status (status time series), image
(image properties) and detection (classifications), plus the
image_with_status view as the portal's migration target. A repeatable,
batched migration splits existing metadata/category rows into it.

The legacy poller is unchanged and keeps writing metadata/category until
the portal is switched over.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-09-18 13:54:21 +00:00
co-authored by Claude Opus 5
parent 94bfc60fc4
commit 2fc8dea10c
13 changed files with 1638 additions and 11 deletions
+179
View File
@@ -0,0 +1,179 @@
-- Normalisiertes Fachmodell: Kamerastatus, Bildeigenschaften, Objektklassifikationen.
-- Idempotent; wird vom gundi-receiver beim Start unter Advisory-Lock angewendet.
-- Setzt das bestehende Schema remote_cam voraus (die App-Rolle hat kein CREATE auf DB-Ebene).
-- Statusupdates / Kameraeigenschaften: eine Zeile je Statusmeldung (Zeitreihe).
CREATE TABLE IF NOT EXISTS remote_cam.camera_status (
status_id bigserial PRIMARY KEY,
imei text NOT NULL,
reported_at timestamptz NOT NULL,
received_at timestamptz NOT NULL DEFAULT now(),
iccid text,
init_key text,
camera_name text,
camera_model text,
firmware_version text,
camera_language text,
customer_timezone text,
mcc text,
mnc text,
lac text,
cell_id text,
camera_network text,
battery_pct integer,
signal_strength integer,
temperature_c integer,
coordinate_switch boolean,
work_period boolean,
work_start text,
work_end text,
image_format text,
image_night text,
thumbnail_size text,
pir_interval text,
time_scan text,
lat double precision,
lon double precision,
settings_changed_at timestamptz,
source text NOT NULL CHECK (source IN ('gundi', 'entrance')),
source_ref text,
raw jsonb
);
-- Legacy hat 173 doppelte (imei, servertime)-Paare, daher keine globale Eindeutigkeit.
CREATE UNIQUE INDEX IF NOT EXISTS camera_status_gundi_uq
ON remote_cam.camera_status (imei, reported_at) WHERE source = 'gundi';
CREATE UNIQUE INDEX IF NOT EXISTS camera_status_entrance_uq
ON remote_cam.camera_status (source_ref) WHERE source = 'entrance';
CREATE INDEX IF NOT EXISTS camera_status_imei_reported_idx
ON remote_cam.camera_status (imei, reported_at DESC);
-- Bildeigenschaften.
CREATE TABLE IF NOT EXISTS remote_cam.image (
image_uuid uuid PRIMARY KEY,
imei text NOT NULL,
servertime timestamptz NOT NULL,
ocr_ts timestamptz,
ocr_state text NOT NULL DEFAULT 'PENDING' CHECK (ocr_state IN ('PENDING', 'OK', 'FAIL')),
ocr_attempts integer NOT NULL DEFAULT 0,
ocr_error text,
camera_ts timestamptz,
camera_name text,
camera_make text,
lat double precision,
lon double precision,
size_bytes integer,
processed_key text,
thumbnail_key text,
source text NOT NULL CHECK (source IN ('gundi', 'entrance')),
source_ref text,
imported_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
raw jsonb
);
CREATE INDEX IF NOT EXISTS image_imei_servertime_idx
ON remote_cam.image (imei, servertime DESC);
CREATE INDEX IF NOT EXISTS image_servertime_idx
ON remote_cam.image (servertime DESC);
CREATE INDEX IF NOT EXISTS image_ocr_pending_idx
ON remote_cam.image (updated_at) WHERE ocr_state = 'PENDING';
-- Objektklassifikationen.
CREATE TABLE IF NOT EXISTS remote_cam.detection (
detection_id bigserial PRIMARY KEY,
image_uuid uuid NOT NULL REFERENCES remote_cam.image (image_uuid) ON DELETE CASCADE,
category text NOT NULL,
confidence double precision CHECK (confidence BETWEEN 0 AND 1),
bbox_x integer,
bbox_y integer,
bbox_w integer,
bbox_h integer,
model_version text,
source text NOT NULL CHECK (source IN ('gundi', 'entrance'))
);
CREATE INDEX IF NOT EXISTS detection_image_confidence_idx
ON remote_cam.detection (image_uuid, confidence DESC);
CREATE INDEX IF NOT EXISTS detection_category_idx
ON remote_cam.detection (category);
-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine.
CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox (
object_id uuid PRIMARY KEY,
event_type text NOT NULL,
source text,
recorded_at timestamptz,
payload jsonb NOT NULL,
received_at timestamptz NOT NULL DEFAULT now(),
attachment_key text,
attachment_received_at timestamptz,
state text NOT NULL CHECK (state IN (
'AWAITING_ATTACHMENT', 'RECEIVED', 'INGESTED',
'FAILED', 'ABANDONED', 'IGNORED')),
attempts integer NOT NULL DEFAULT 0,
last_error text,
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS gundi_inbox_open_idx
ON remote_cam.gundi_inbox (state, updated_at)
WHERE state IN ('AWAITING_ATTACHMENT', 'RECEIVED');
-- Bild + der zum Aufnahme-Serverzeitpunkt gueltige Kamerastatus.
-- Tie-Breaker bei gleichem reported_at: der aus derselben Legacy-Zeile abgespaltene Status.
CREATE OR REPLACE VIEW remote_cam.image_with_status AS
SELECT
i.image_uuid,
i.imei,
i.servertime,
i.ocr_ts,
i.ocr_state,
i.camera_ts,
i.camera_name,
i.camera_make,
i.lat,
i.lon,
i.size_bytes,
i.processed_key,
i.thumbnail_key,
i.source,
i.imported_at,
s.status_id,
s.reported_at AS status_reported_at,
s.iccid,
s.init_key,
s.camera_name AS status_camera_name,
s.camera_model,
s.firmware_version,
s.camera_language,
s.customer_timezone,
s.mcc,
s.mnc,
s.lac,
s.cell_id,
s.camera_network,
s.battery_pct,
s.signal_strength,
s.temperature_c,
s.coordinate_switch,
s.work_period,
s.work_start,
s.work_end,
s.image_format,
s.image_night,
s.thumbnail_size,
s.pir_interval,
s.time_scan,
s.settings_changed_at
FROM remote_cam.image i
LEFT JOIN LATERAL (
SELECT cs.*
FROM remote_cam.camera_status cs
WHERE cs.imei = i.imei
AND cs.reported_at <= i.servertime
ORDER BY cs.reported_at DESC,
(cs.source_ref = i.image_uuid::text) DESC NULLS LAST,
cs.status_id DESC
LIMIT 1
) s ON TRUE;
+94
View File
@@ -0,0 +1,94 @@
-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt
-- ein image und ein camera_status; remote_cam.category wird zu detection.
--
-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py
-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen.
-- Alttabellen werden nur gelesen.
-- step: image
INSERT INTO remote_cam.image (
image_uuid, imei, servertime, ocr_ts, ocr_state, ocr_error, camera_ts,
camera_name, camera_make, lat, lon, size_bytes, processed_key, thumbnail_key,
source, source_ref, imported_at, updated_at
)
SELECT
m.metadata_uuid::uuid,
m.imei,
m.servertime,
m.ocr_ts,
CASE WHEN m.ocr_ts IS NULL THEN 'FAIL' ELSE 'OK' END,
CASE WHEN m.ocr_ts IS NULL THEN 'legacy: no ocr_ts' END,
m."ImageCreationDate",
m."CameraName",
m."CameraModel",
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Latitude" END,
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Longitude" END,
m."SizeInByte",
j.processed_jpg_key,
j.thumbnail_key,
'entrance',
m.metadata_uuid,
COALESCE(m.import_ts, now()),
now()
FROM _mig_batch b
JOIN remote_cam.metadata m ON m.metadata_uuid = b.metadata_uuid
LEFT JOIN remote_cam.import_job j ON j.image_uuid = m.metadata_uuid
WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
ON CONFLICT (image_uuid) DO NOTHING;
-- step: camera_status
INSERT INTO remote_cam.camera_status (
imei, reported_at, received_at, iccid, init_key, camera_name, camera_model,
firmware_version, camera_language, customer_timezone, mcc, mnc, lac, camera_network,
battery_pct, signal_strength, temperature_c, coordinate_switch, work_period,
work_start, work_end, image_format, image_night, thumbnail_size, pir_interval, time_scan,
lat, lon, settings_changed_at, source, source_ref
)
SELECT
m.imei,
m.servertime,
COALESCE(m.import_ts, now()),
m.iccid,
m."INITKey",
m."CameraName",
m."CameraModel",
m."FirmwareVersion",
m."CameraLanguage",
m."CustomerTimezone",
m."MobileCountryCode",
m."MobileNetworkCode",
m."LocationAreaCode",
m."CameraNetwork",
m."BatteryStatus",
m."SignalStrength",
m."Temperature",
m."CoordinateSwitch",
m."WorkPeriod",
m."WorkStart",
m."WorkEnd",
m."ImageFormat",
m."ImageNight",
m."ThumbnailSize",
m."PIRInterval",
m."TimeScan",
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Latitude" END,
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Longitude" END,
m."DateTimeLastSettings",
'entrance',
m.metadata_uuid
FROM _mig_batch b
JOIN remote_cam.metadata m ON m.metadata_uuid = b.metadata_uuid
WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING;
-- step: detection
INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source)
SELECT i.image_uuid, c.category, c.score, c.version, 'entrance'
FROM _mig_batch b
JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid
JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid
WHERE c.category IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM remote_cam.detection d
WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance'
);