Compare commits
10
Commits
2e9ee3e014
..
gundi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9dbb8430ec | ||
|
|
39b22f0124 | ||
|
|
ed105fbff8 | ||
|
|
e7e5b6fb87 | ||
|
|
6d518b4542 | ||
|
|
cb2566a537 | ||
|
|
0f574cccde | ||
|
|
2fc8dea10c | ||
|
|
94bfc60fc4 | ||
|
|
88ef0d6943 |
@@ -1,5 +0,0 @@
|
||||
PG_DSN=postgresql://postgres:DFGk5s9H21lKao2K@136.243.41.58:7777/meles
|
||||
S3_BUCKET=trapper-meles
|
||||
S3_ACCESS_KEY=UG950FCGWFYRXSXXVUAH
|
||||
S3_SECRET_KEY=rXpspNhboZY6zNZi7djjq2QaXPA4uwsO9jXf4AXk
|
||||
S3_ENDPOINT=https://fsn1.your-objectstorage.com/
|
||||
@@ -0,0 +1,27 @@
|
||||
# .env.example - Vorlage fuer die Secrets.
|
||||
#
|
||||
# Auf dem Server eine echte .env daraus erzeugen und mit den NEU ROTIERTEN
|
||||
# Werten befuellen:
|
||||
# cp .env.example .env
|
||||
# chmod 600 .env
|
||||
# Die echte .env wird durch .gitignore ausgeschlossen und darf NIE eingecheckt werden.
|
||||
|
||||
# --- Hetzner Object Storage (S3) ---
|
||||
S3_ENDPOINT=https://fsn1.your-objectstorage.com/
|
||||
S3_BUCKET=trapper-meles
|
||||
S3_ACCESS_KEY=__NEUEN_ACCESS_KEY_EINTRAGEN__
|
||||
S3_SECRET_KEY=__NEUEN_SECRET_KEY_EINTRAGEN__
|
||||
|
||||
# --- PostgreSQL ---
|
||||
# TLS erzwingen (sslmode=require). Host/Port zeigen auf den NEUEN DB-Zugang
|
||||
# (idealerweise ueber privates Netz / SSH-Tunnel, nicht oeffentlich erreichbar).
|
||||
PG_DSN=postgresql://USER:PASSWORT@HOST:PORT/meles?sslmode=require
|
||||
|
||||
# --- DeerMapper API (nur noetig, wenn app.enable_deermapper_api: true) ---
|
||||
DEERMAPPER_API_URL=https://webapp.deermapper.net/api/icu
|
||||
DEERMAPPER_API_KEY=__NEUEN_API_KEY_EINTRAGEN__
|
||||
|
||||
# --- Gundi-Receiver (Push-Schnittstelle fuer ICU) ---
|
||||
# Diesen Wert bekommt ICU als "apikey". Mind. 24 Zeichen, z.B.:
|
||||
# python3 -c "import secrets; print(secrets.token_urlsafe(32))"
|
||||
GUNDI_API_KEY=__NEUEN_API_KEY_EINTRAGEN__
|
||||
@@ -1,3 +1,11 @@
|
||||
*.pyc
|
||||
*.pyo
|
||||
__pycache__/
|
||||
|
||||
# Secrets - NIEMALS einchecken. Echte Werte leben nur in der lokalen .env auf dem Server.
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Optionale lokale Config-Overrides (falls verwendet)
|
||||
config.local.yaml
|
||||
|
||||
+11
-2
@@ -15,6 +15,15 @@ WORKDIR /app
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY main.py /app/main.py
|
||||
# Code + nicht-geheime Konfiguration ins Image (Secrets kommen zur Laufzeit via ENV).
|
||||
COPY main.py gundi_pipeline.py gundi_receiver.py /app/
|
||||
COPY schema/ /app/schema/
|
||||
COPY tools/ /app/tools/
|
||||
COPY config.yaml /app/config.yaml
|
||||
|
||||
CMD ["python", "/app/main.py"]
|
||||
# Haertung: nicht als root laufen.
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
USER appuser
|
||||
|
||||
# Standard ist der Gundi-Receiver; der S3-Poller (main.py) ist seit 2026-10-16 stillgelegt.
|
||||
CMD ["python", "/app/gundi_receiver.py"]
|
||||
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
# Migration & Secret-Rotation - Runbook
|
||||
|
||||
Ziel: `melesICUmover` sicher vom **Altserver** auf den **neuen Server** umziehen,
|
||||
alle kompromittierten Zugangsdaten **rotieren**, den Altserver **stilllegen** und
|
||||
die Secrets aus der **Git-History** entfernen.
|
||||
|
||||
> Kontext: Der Poller haelt keine eigenen Daten - Bilder liegen in Hetzner S3
|
||||
> (`trapper-meles`), der Zustand in PostgreSQL. "Migration" heisst also: den
|
||||
> **Worker-Container** umziehen und die **Credentials** erneuern. Es gibt keine
|
||||
> Datenmigration.
|
||||
|
||||
## Warum rotieren (nicht nur entfernen)?
|
||||
|
||||
Folgende Secrets lagen im Klartext in `config.yaml` und `.env` **und in der
|
||||
Git-History** (Commits `400f8a5`/`b35394c`) auf `git.meles.eu`. Sie gelten als
|
||||
kompromittiert und muessen ersetzt werden:
|
||||
|
||||
- S3 Access Key + Secret Key (Hetzner Object Storage)
|
||||
- PostgreSQL-Passwort (`postgres@136.243.41.58:7777`)
|
||||
- DeerMapper API-Key
|
||||
|
||||
Das Bereinigen des Codes allein schuetzt nicht - die alten Werte bleiben gueltig,
|
||||
bis sie widerrufen/geaendert werden.
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 - Vorbereitung (kein Ausfall)
|
||||
|
||||
1. Auf dem **neuen Server** Docker + Compose pruefen/installieren:
|
||||
```bash
|
||||
docker --version && docker compose version || curl -fsSL https://get.docker.com | sh
|
||||
```
|
||||
2. Repo auf den neuen Server holen (bereinigter Stand, ohne Secrets):
|
||||
```bash
|
||||
git clone ssh://git.meles.eu/meles2/melesICUmover.git
|
||||
cd melesICUmover
|
||||
```
|
||||
3. `.env` anlegen (noch NICHT starten):
|
||||
```bash
|
||||
cp .env.example .env
|
||||
chmod 600 .env
|
||||
```
|
||||
|
||||
## Phase 1 - S3-Key rotieren (kein Ausfall, additiv)
|
||||
|
||||
Hetzner erlaubt mehrere S3-Credentials pro Projekt. Neuen Key **zusaetzlich**
|
||||
anlegen, alten vorerst aktiv lassen:
|
||||
|
||||
1. In der Hetzner Console -> Object Storage -> neuen S3-Zugang (Access/Secret) erzeugen.
|
||||
2. In der `.env` auf dem neuen Server `S3_ACCESS_KEY` / `S3_SECRET_KEY` eintragen.
|
||||
|
||||
## Phase 2 - DeerMapper API-Key rotieren
|
||||
|
||||
Nur relevant, wenn `enable_deermapper_api: true` (aktuell `false`). Falls genutzt:
|
||||
neuen Key beim DeerMapper-Betreiber anfordern und in die `.env` eintragen. Sonst
|
||||
Feld leer lassen.
|
||||
|
||||
## Phase 3 - Cutover (kurzer, geplanter Ausfall)
|
||||
|
||||
Der PostgreSQL-Passwortwechsel trennt sofort den Altserver - daher hier gebuendelt:
|
||||
|
||||
1. **Altserver-Poller stoppen** (kein Doppelbetrieb auf denselben Bucket/DB):
|
||||
```bash
|
||||
# auf dem ALTSERVER
|
||||
docker compose down # bzw. docker stop melesicumover
|
||||
```
|
||||
2. **PostgreSQL-Passwort aendern** (Beispiel; besser: eigene DB-Rolle fuer den Dienst):
|
||||
```sql
|
||||
ALTER USER postgres WITH PASSWORD '<NEUES_STARKES_PASSWORT>';
|
||||
-- Empfohlen stattdessen: dedizierte Rolle mit minimalen Rechten auf remote_cam.*
|
||||
-- CREATE ROLE icu_mover LOGIN PASSWORD '...';
|
||||
-- GRANT USAGE ON SCHEMA remote_cam TO icu_mover;
|
||||
-- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA remote_cam TO icu_mover;
|
||||
```
|
||||
3. `PG_DSN` in der `.env` des neuen Servers setzen - **mit `?sslmode=require`** und,
|
||||
sofern moeglich, ueber privates Netz / SSH-Tunnel statt der oeffentlichen IP.
|
||||
4. **Neuen Server starten und verifizieren:**
|
||||
```bash
|
||||
# auf dem NEUEN Server
|
||||
docker compose up -d --build
|
||||
docker compose logs -f # auf [RESULT] ...-Zeilen achten
|
||||
```
|
||||
Verifikation:
|
||||
- Log zeigt verarbeitete UUIDs mit `result=success`.
|
||||
- In S3 landen neue Objekte unter `icu/processed/` und `icu/thumbnails/`.
|
||||
- `icu/entrance/` wird geleert (Cleanup laeuft).
|
||||
- DB: `SELECT status, count(*) FROM remote_cam.import_job GROUP BY 1;`
|
||||
|
||||
## Phase 4 - Altzugaenge widerrufen
|
||||
|
||||
1. Alten S3-Key in der Hetzner Console **loeschen** (erst nachdem der neue Server
|
||||
nachweislich laeuft).
|
||||
2. Sicherstellen, dass der Altserver den neuen DB-Zugang **nicht** kennt.
|
||||
|
||||
## Phase 5 - DB-Zugang absichern (Haertung)
|
||||
|
||||
- PostgreSQL **nicht** auf `0.0.0.0:7777` oeffentlich anbieten. Firewall so setzen,
|
||||
dass nur die IP des neuen Servers (oder ein privates Netz) den DB-Port erreicht.
|
||||
- TLS erzwingen (`sslmode=require`), Hetzner Cloud Firewall bzw. `pg_hba.conf` pruefen.
|
||||
|
||||
## Phase 6 - Altserver stilllegen / neu aufsetzen
|
||||
|
||||
1. Sicherstellen: keine unersetzten Daten mehr lokal (der Poller hat keine).
|
||||
2. Container + Images + lokale `.env`/`config.yaml` mit Secrets entfernen:
|
||||
```bash
|
||||
# auf dem ALTSERVER
|
||||
docker compose down --rmi all --volumes
|
||||
shred -u .env 2>/dev/null || rm -f .env
|
||||
```
|
||||
3. Server neu aufsetzen bzw. deprovisionieren. Falls neu installiert: alte
|
||||
SSH-/Deploy-Keys, die auf `git.meles.eu` Zugriff hatten, am Git-Server widerrufen.
|
||||
|
||||
## Phase 7 - Git-History bereinigen (koordiniert, destruktiv)
|
||||
|
||||
Erst **nach** erfolgreicher Rotation - danach sind die alten Werte ohnehin wertlos,
|
||||
aber sie sollen auch nicht mehr auffindbar sein.
|
||||
|
||||
> Warnung: Das schreibt die History um und erfordert `--force`-Push. Alle Personen
|
||||
> mit Klon muessen anschliessend neu klonen. Vorher mit dem Team abstimmen.
|
||||
|
||||
Mit [git filter-repo](https://github.com/newren/git-filter-repo) (empfohlen):
|
||||
|
||||
```bash
|
||||
# frisches Spiegel-Klon als Arbeitskopie
|
||||
git clone ssh://git.meles.eu/meles2/melesICUmover.git repo-clean
|
||||
cd repo-clean
|
||||
|
||||
# .env komplett aus der History entfernen
|
||||
git filter-repo --path .env --invert-paths
|
||||
|
||||
# Secret-Werte auch aus historischen config.yaml-Versionen tilgen.
|
||||
# Die vier ALTEN Werte NICHT hier ins Repo schreiben - die Datei liegt ausserhalb
|
||||
# des Repos und wird nach dem Lauf geloescht. Werte aus der alten .env / History
|
||||
# (git show 400f8a5:config.yaml) entnehmen:
|
||||
cat > ../replacements.txt <<'EOF'
|
||||
<ALTES_PG_PASSWORT>==>ENTFERNT
|
||||
<ALTES_S3_SECRET_KEY>==>ENTFERNT
|
||||
<ALTER_S3_ACCESS_KEY>==>ENTFERNT
|
||||
<ALTER_DEERMAPPER_API_KEY>==>ENTFERNT
|
||||
EOF
|
||||
git filter-repo --replace-text ../replacements.txt
|
||||
|
||||
# Remote neu setzen und History ueberschreiben
|
||||
git remote add origin ssh://git.meles.eu/meles2/melesICUmover.git
|
||||
git push --force --all origin
|
||||
git push --force --tags origin
|
||||
```
|
||||
|
||||
Danach `replacements.txt` loeschen. Alternative: BFG Repo-Cleaner.
|
||||
|
||||
---
|
||||
|
||||
## Rollback
|
||||
|
||||
Falls der neue Server Probleme macht, **bevor** Phase 4/6 abgeschlossen sind:
|
||||
alten S3-Key noch aktiv lassen, altes PG-Passwort noch nicht wegwerfen, und den
|
||||
Altserver-Poller wieder starten (`docker compose up -d`). Deshalb Altzugaenge erst
|
||||
in Phase 4 widerrufen.
|
||||
|
||||
## Abschluss-Checkliste
|
||||
|
||||
- [ ] Neuer Server verarbeitet Bilder (`result=success`, S3 + DB aktualisiert)
|
||||
- [ ] Neue S3-Keys aktiv, alte geloescht
|
||||
- [ ] Neues PG-Passwort/-Rolle aktiv, `sslmode=require`, DB-Port nicht oeffentlich
|
||||
- [ ] DeerMapper-Key rotiert (falls genutzt)
|
||||
- [ ] Altserver-Poller gestoppt, `.env` sicher geloescht, Server neu aufgesetzt
|
||||
- [ ] Alte SSH-/Deploy-Keys am Git-Server widerrufen
|
||||
- [ ] Git-History bereinigt + force-push, Team informiert
|
||||
- [ ] Repo enthaelt keine Secrets mehr (`git grep` auf HEAD + Stichprobe History)
|
||||
@@ -1 +1,192 @@
|
||||
# melesICUmover
|
||||
# melesICUmover
|
||||
|
||||
> **Stand 2026-10-16:** Bilder und Kamerastatus kommen ausschliesslich ueber den
|
||||
> Gundi-Push (Dienst `gundi-receiver`, Abschnitt "Gundi"). Der unten beschriebene
|
||||
> S3-Poller ist stillgelegt und nicht mehr in `docker-compose.yaml`; seine Tabellen
|
||||
> (`metadata`, `category`, `import_job`) werden nicht mehr befuellt und sind nach
|
||||
> `image` / `camera_status` / `detection` migriert. Die Beschreibung bleibt als Referenz.
|
||||
|
||||
Frueher: Dauer-Poller, der Wildkamera-Bilder aus einem Hetzner-S3-Bucket verarbeitet:
|
||||
liest neue Bilder + `metadata.json` (+ optional `_categories.json`), extrahiert per
|
||||
**OCR** (Tesseract) den Overlay-Zeitstempel, schreibt **EXIF**-Daten (exiftool),
|
||||
legt verarbeitetes Bild + Thumbnail zurueck nach S3, persistiert alles nach
|
||||
**PostgreSQL** (`remote_cam.*`), pusht optional an die **DeerMapper-API** und
|
||||
raeumt den Entrance-Ordner auf. Zustand pro Bild in `remote_cam.import_job.status`.
|
||||
|
||||
## Konfiguration
|
||||
|
||||
Zweigeteilt - bewusst:
|
||||
|
||||
| Datei | Inhalt | Im Git? |
|
||||
|-------|--------|---------|
|
||||
| `config.yaml` | Nicht-geheime App-Tunables + oeffentliche Endpunkte | **ja** (Template) |
|
||||
| `.env` | Secrets: S3-Keys, `PG_DSN`, API-Key | **nein** (`.gitignore`) |
|
||||
|
||||
Secrets werden **ausschliesslich** ueber Umgebungsvariablen geladen; die Werte in
|
||||
`.env` ueberschreiben bzw. ergaenzen `config.yaml`. Beim Start prueft die App die
|
||||
Vollstaendigkeit und bricht mit klarer Meldung ab, falls ein Secret fehlt.
|
||||
|
||||
Unterstuetzte Umgebungsvariablen: `S3_ENDPOINT`, `S3_BUCKET`, `S3_ACCESS_KEY`,
|
||||
`S3_SECRET_KEY`, `PG_DSN`, `DEERMAPPER_API_URL`, `DEERMAPPER_API_KEY`, `GUNDI_API_KEY`,
|
||||
`CONFIG_YAML`, `DEBUG`.
|
||||
|
||||
## Betrieb (Docker)
|
||||
|
||||
```bash
|
||||
cp .env.example .env # danach echte, rotierte Werte eintragen
|
||||
chmod 600 .env
|
||||
docker compose up -d --build gundi-receiver
|
||||
docker compose logs -f gundi-receiver
|
||||
```
|
||||
|
||||
### Stoerungen und Lebenszeichen (Poller)
|
||||
|
||||
Fehler eines Zyklus (S3 5xx/Throttling, Netz, DB weg) beenden den Poller nicht:
|
||||
er loggt eine Zeile `[CYCLE_FAIL] kind=transient ...` und versucht es mit
|
||||
exponentiellem Backoff erneut (`backoff_initial_seconds` bis `backoff_max_seconds`,
|
||||
Reset nach Erfolg, dann `[RECOVERED]`). Unerwartete Fehler (`kind=unexpected`,
|
||||
z. B. AccessDenied oder Programmierfehler) werden zusaetzlich mit Traceback geloggt.
|
||||
Ein Bild, bei dem S3/DB wegbricht, bleibt wie bisher auf `ERROR`; der Rest des
|
||||
Batches wartet auf den naechsten Zyklus.
|
||||
|
||||
Alle `heartbeat_seconds` erscheint `[HEARTBEAT]` mit Zaehlern (Zyklen, Bilder,
|
||||
Fehler, letzter Erfolg). Der Docker-Healthcheck prueft, ob die Hauptschleife
|
||||
vorankommt (`/tmp/melesicumover.heartbeat` juenger als 15 min):
|
||||
|
||||
```bash
|
||||
docker inspect -f '{{.State.Health.Status}} restarts={{.RestartCount}}' melesicumover
|
||||
docker exec melesicumover cat /tmp/melesicumover.heartbeat
|
||||
```
|
||||
|
||||
S3-Timeouts/-Retries stehen unter `app.s3_*` in `config.yaml`.
|
||||
|
||||
### Eingang pruefen (verwaiste Objekte)
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps gundi-receiver python -m tools.entrance_inventory --db
|
||||
```
|
||||
|
||||
Listet `icu/entrance/` nach Typ (vollstaendig, JPG ohne JSON, JSON ohne JPG,
|
||||
nur Kategorien, fremd) und Alter, optional mit `import_job`-Status. Nur lesend.
|
||||
|
||||
### Tests
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python -m unittest discover -s tests -v # oder: python -m pytest tests
|
||||
```
|
||||
|
||||
Simulierte S3-/DB-Stoerungen fuer Backoff und Fehlerbehandlung; weder Netz noch DB noetig.
|
||||
|
||||
## Gundi-Push-Pfad (`gundi-receiver`)
|
||||
|
||||
ICU pusht Events ueber eine Gundi-v2-kompatible API. Der Service `gundi-receiver`
|
||||
(gleiches Image, `gundi_receiver.py`) nimmt sie entgegen:
|
||||
|
||||
| Route | Zweck |
|
||||
|-------|-------|
|
||||
| `POST /v2/events/` | `camera_status_report` oder `cameratrap_rep` / `icu_cameratrap_rep` (JSON, Objekt oder Liste) |
|
||||
| `POST /v2/events/<object_id>/attachments/` | JPEG zum `cameratrap_rep` (multipart/form-data) |
|
||||
| `GET /healthz` | Healthcheck |
|
||||
|
||||
Auth ueber den Header `apikey` = `GUNDI_API_KEY` aus der `.env`.
|
||||
|
||||
Ablauf: Event und JPEG werden durable abgelegt und sofort quittiert. Stufe 1
|
||||
(Sekunden) schreibt processed-JPEG + Thumbnail und `image` + `detection`. Stufe 2
|
||||
liest danach asynchron per OCR den Kamera-Zeitstempel, setzt `image.ocr_ts` und
|
||||
`DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert
|
||||
Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
|
||||
|
||||
### Verhalten bei S3-/DB-Ausfall
|
||||
|
||||
| Teil | Verhalten |
|
||||
|------|-----------|
|
||||
| Status-Events, Event anlegen | brauchen kein S3; bei DB-Ausfall `503` + `Retry-After: 30` |
|
||||
| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen |
|
||||
| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` |
|
||||
| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) |
|
||||
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder in den letzten 24 h neue `FAILED`/`FAIL` entstanden sind (aeltere loesen keinen Dauer-Alarm aus) |
|
||||
|
||||
Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`):
|
||||
|
||||
```bash
|
||||
docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue --apply
|
||||
```
|
||||
|
||||
Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne Wiederholung geht das Bild verloren, wenn S3 laenger ausfaellt.
|
||||
|
||||
### Datenmodell (`schema/001_domain_model.sql`)
|
||||
|
||||
| Tabelle | Inhalt |
|
||||
|---------|--------|
|
||||
| `remote_cam.camera_status` | Statusupdates / Kameraeigenschaften (Zeitreihe je IMEI) |
|
||||
| `remote_cam.image` | Bildeigenschaften inkl. `servertime`, `ocr_ts`, `ocr_state` |
|
||||
| `remote_cam.detection` | Objektklassifikationen, `confidence` 0..1, Rohbezeichnung der Quelle |
|
||||
| `remote_cam.category_label` / View `detection_labeled` | einheitliches Label je Rohbezeichnung (`Deer-Like` -> `deer`) |
|
||||
| `remote_cam.image_superseded` | entfernte Gundi-Zwillinge aus dem Parallelbetrieb (mit Originalzeile) |
|
||||
| `remote_cam.image_with_status` | View: Bild + zum `servertime` gueltiger Status |
|
||||
| `remote_cam.gundi_inbox` | Transportschicht: Rohevents + Zustand des Push-Pfads |
|
||||
|
||||
Das Schema legt der Receiver beim Start an. Die Legacy-Tabellen `metadata` /
|
||||
`category` / `import_job` werden seit dem Stilllegen des Pollers (2026-10-16) nicht
|
||||
mehr befuellt; das Portal liest seitdem nur noch das Fachmodell.
|
||||
|
||||
### Bestandsdaten migrieren
|
||||
|
||||
Split `metadata` -> `image` + `camera_status`, `category` -> `detection`
|
||||
(nur Klassen ab Score 0.5, ohne `''`/`none`). Wiederholbar, batchweise, liest die
|
||||
Alttabellen nur:
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --plan # nur lesen
|
||||
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy
|
||||
```
|
||||
|
||||
Im Parallelbetrieb kommt dasselbe Bild auch ueber Gundi (gleiche imei, gleiche
|
||||
Dateigroesse, <= 10 min). Nach der Migration den Gundi-Zwilling entfernen; behalten
|
||||
wird der Legacy-Zwilling, an dem die KI-Ergebnisse haengen:
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --dedupe --dry-run
|
||||
docker compose run --rm --no-deps gundi-receiver python -m tools.migrate_legacy --dedupe
|
||||
```
|
||||
|
||||
Vor dem Portal-Cutover und nach dem Stoppen des Pollers beides erneut ausfuehren.
|
||||
`--verify-only` prueft nur die Zaehlungen.
|
||||
|
||||
### Test
|
||||
|
||||
```bash
|
||||
docker compose up -d --build gundi-receiver
|
||||
docker compose run --rm --no-deps gundi-receiver \
|
||||
python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg /pfad/zu/bild.jpg
|
||||
```
|
||||
|
||||
Der Pfad schreibt unter `icu/gundi/` (siehe `gundi:` in `config.yaml`); bis
|
||||
2026-10-16 war es `icu/gundi-test/`. Der genaue Schluessel steht je Bild in
|
||||
`image.processed_key` / `thumbnail_key` - Konsumenten bauen ihn nicht selbst zusammen.
|
||||
|
||||
### nginx (`icu.meles.eu`)
|
||||
|
||||
vhost liegt unter [`deploy/nginx/icu.meles.eu`](deploy/nginx/icu.meles.eu):
|
||||
|
||||
```bash
|
||||
sudo cp deploy/nginx/icu.meles.eu /etc/nginx/sites-available/
|
||||
sudo ln -s /etc/nginx/sites-available/icu.meles.eu /etc/nginx/sites-enabled/
|
||||
sudo nginx -t && sudo systemctl reload nginx
|
||||
sudo certbot --nginx -d icu.meles.eu --redirect
|
||||
```
|
||||
|
||||
ICU-Endpunkt: `https://icu.meles.eu/v2/events/` (immer https — der 80->443-Redirect
|
||||
ist ein 301 und wuerde POST in GET verwandeln).
|
||||
|
||||
## Sicherheitshinweise
|
||||
|
||||
- **Niemals** Secrets in `config.yaml` oder ins Git schreiben - nur in `.env`.
|
||||
- `PG_DSN` mit `?sslmode=require`; die Datenbank sollte **nicht** oeffentlich
|
||||
erreichbar sein (privates Netz / Firewall / SSH-Tunnel).
|
||||
- Debug-Port (`5678`) ist standardmaessig aus; bei Bedarf nur an `127.0.0.1` binden.
|
||||
- Nach Kompromittierung (z.B. Secrets waren im Git): **rotieren**, nicht nur entfernen.
|
||||
|
||||
Siehe [`MIGRATION.md`](MIGRATION.md) fuer Secret-Rotation, Umzug auf einen neuen
|
||||
Server und Stilllegung des Altservers.
|
||||
|
||||
+44
-6
@@ -1,16 +1,22 @@
|
||||
# config.yaml (Beispiel)
|
||||
# config.yaml - Nicht-geheime Konfiguration (App-Tunables + oeffentliche Endpunkte).
|
||||
#
|
||||
# WICHTIG: Hier gehoeren KEINE Zugangsdaten hinein.
|
||||
# Secrets kommen ausschliesslich aus Umgebungsvariablen (.env):
|
||||
# S3_ACCESS_KEY, S3_SECRET_KEY, PG_DSN, DEERMAPPER_API_KEY
|
||||
# (optional koennen auch S3_ENDPOINT / S3_BUCKET / DEERMAPPER_API_URL per ENV
|
||||
# ueberschrieben werden.)
|
||||
|
||||
s3:
|
||||
endpoint: "https://fsn1.your-objectstorage.com/"
|
||||
access_key: "UG950FCGWFYRXSXXVUAH"
|
||||
secret_key: "rXpspNhboZY6zNZi7djjq2QaXPA4uwsO9jXf4AXk"
|
||||
bucket: "trapper-meles"
|
||||
# access_key / secret_key -> via ENV (.env): S3_ACCESS_KEY / S3_SECRET_KEY
|
||||
|
||||
postgres:
|
||||
dsn: "postgresql://postgres:DFGk5s9H21lKao2K@136.243.41.58:7777/meles"
|
||||
postgres: {}
|
||||
# dsn -> via ENV (.env): PG_DSN (inkl. ?sslmode=require)
|
||||
|
||||
deermapper-api:
|
||||
base_url: "https://webapp.deermapper.net/api/icu"
|
||||
apiKey: "695bc217-3b40-48bb-bb12-17fc5b08b320"
|
||||
# apiKey -> via ENV (.env): DEERMAPPER_API_KEY
|
||||
|
||||
app:
|
||||
entrance_prefix: "icu/entrance/"
|
||||
@@ -33,3 +39,35 @@ app:
|
||||
enable_deermapper_api: false
|
||||
deermapper_api_timeout_seconds: 20
|
||||
deermapper_api_image_field: "image"
|
||||
# Robustheit (nur Poller): S3-Timeouts/Retries, Backoff bei Stoerungen, Lebenszeichen
|
||||
s3_connect_timeout_seconds: 10
|
||||
s3_read_timeout_seconds: 30
|
||||
s3_retry_mode: "standard" # standard | adaptive | legacy
|
||||
s3_max_attempts: 4 # inkl. erstem Versuch
|
||||
pg_connect_timeout_seconds: 10
|
||||
backoff_initial_seconds: 5 # 5 s, 10 s, 20 s ... nach fehlgeschlagenem Zyklus
|
||||
backoff_max_seconds: 300 # hoechstens 5 min; Reset nach erfolgreichem Zyklus
|
||||
heartbeat_seconds: 300 # [HEARTBEAT]-Logzeile mit Zaehlern
|
||||
|
||||
# Gundi-v2-Push-Empfaenger (Service gundi-receiver).
|
||||
# api key -> via ENV (.env): GUNDI_API_KEY (min. 24 Zeichen)
|
||||
gundi:
|
||||
enabled: true
|
||||
bind_host: "0.0.0.0" # im Container; nach aussen nur 127.0.0.1:8090 (compose)
|
||||
bind_port: 8090
|
||||
inbox_prefix: "icu/gundi/inbox/"
|
||||
processed_prefix: "icu/gundi/processed/" # Portal liest den Schluessel aus image.processed_key
|
||||
thumb_prefix: "icu/gundi/thumbnails/" # Portal liest den Schluessel aus image.thumbnail_key
|
||||
workers: 4
|
||||
ocr_workers: 2
|
||||
max_request_bytes: 33554432
|
||||
abandon_after_hours: 24
|
||||
sweep_seconds: 60
|
||||
stale_after_seconds: 300
|
||||
max_attempts: 3
|
||||
# S3-Ausfall-Robustheit (siehe README): Upload-Request scheitert schnell mit 503 + Retry-After;
|
||||
# S3-/DB-Stoerungen zaehlen nicht gegen max_attempts (nur Datenfehler).
|
||||
upload_connect_timeout_seconds: 5
|
||||
upload_read_timeout_seconds: 30
|
||||
upload_max_attempts: 2 # inkl. erstem Versuch
|
||||
alarm_after_seconds: 1800 # [GUNDI][ALARM], wenn Eintraege laenger haengen
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# nginx-vhost fuer den Gundi-Receiver. Nur HTTP; TLS ergaenzt certbot --nginx.
|
||||
# ICU muss direkt https:// verwenden: der 80->443-Redirect von certbot ist ein 301
|
||||
# und wuerde POST in GET verwandeln.
|
||||
server {
|
||||
listen 80;
|
||||
server_name icu.meles.eu;
|
||||
|
||||
client_max_body_size 32m; # = gundi.max_request_bytes
|
||||
|
||||
location /v2/events {
|
||||
proxy_pass http://127.0.0.1:8090;
|
||||
include /etc/nginx/proxy_params;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
location = /healthz {
|
||||
proxy_pass http://127.0.0.1:8090;
|
||||
include /etc/nginx/proxy_params;
|
||||
}
|
||||
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
+22
-6
@@ -1,13 +1,29 @@
|
||||
services:
|
||||
melesicumover:
|
||||
# Der fruehere S3-Poller (Dienst melesicumover, CMD des Images: main.py) ist seit
|
||||
# 2026-10-16 stillgelegt; alle Bilder kommen ueber den Gundi-Push. main.py bleibt
|
||||
# als Bibliothek (Konfiguration, S3, OCR, EXIF) fuer den Receiver im Image.
|
||||
#
|
||||
# Debugging nur bei Bedarf, lokal: DEBUG auf "1" und einen Port NUR an localhost
|
||||
# binden (z. B. "127.0.0.1:5678:5678").
|
||||
|
||||
# Gundi-v2-Push-Empfaenger. Nur an localhost gebunden - TLS und oeffentliche
|
||||
# Erreichbarkeit uebernimmt nginx (icu.meles.eu). Secrets aus der lokalen .env
|
||||
# (nicht im Image, nicht im Git).
|
||||
gundi-receiver:
|
||||
build: .
|
||||
container_name: melesicumover
|
||||
container_name: melesicumover-gundi
|
||||
restart: unless-stopped
|
||||
command: ["python", "/app/gundi_receiver.py"]
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
CONFIG_YAML: /app/config.yaml
|
||||
DEBUG: "0"
|
||||
ports:
|
||||
- "5678:5678"
|
||||
volumes:
|
||||
- ./:/app
|
||||
- ./config.yaml:/app/config.yaml:ro
|
||||
- "127.0.0.1:8090:8090"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/healthz', timeout=3)"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
|
||||
@@ -0,0 +1,780 @@
|
||||
"""Push-Pfad (Gundi v2): Mapping ins Fachmodell, Ingest (Stufe 1), OCR (Stufe 2), Sweep."""
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid as uuidlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import psycopg
|
||||
import yaml
|
||||
from botocore.config import Config as BotoClientConfig
|
||||
from dateutil import tz
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from main import (
|
||||
AppConfig,
|
||||
S3,
|
||||
exif_write_with_exiftool,
|
||||
is_transient_error,
|
||||
make_thumbnail_bytes,
|
||||
ocr_extract_timestamp,
|
||||
to_exif_datetime,
|
||||
)
|
||||
|
||||
SCHEMA_FILE = Path(__file__).resolve().parent / "schema" / "001_domain_model.sql"
|
||||
SCHEMA_LOCK_KEY = 7_311_000_001
|
||||
UUID_STRICT_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
|
||||
# ICU sendet "icu_cameratrap_rep" (Gundi/EarthRanger-Standard waere "cameratrap_rep"); beide gelten als Bild-Event.
|
||||
EVENT_CAMERATRAP_TYPES = ("cameratrap_rep", "icu_cameratrap_rep")
|
||||
EVENT_STATUS = "camera_status_report"
|
||||
MIN_API_KEY_LEN = 24
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Config
|
||||
# -----------------------
|
||||
@dataclass
|
||||
class GundiConfig:
|
||||
enabled: bool
|
||||
api_key: str
|
||||
bind_host: str = "0.0.0.0"
|
||||
bind_port: int = 8090
|
||||
inbox_prefix: str = "icu/gundi/inbox/"
|
||||
processed_prefix: str = "icu/gundi/processed/"
|
||||
thumb_prefix: str = "icu/gundi/thumbnails/"
|
||||
workers: int = 4
|
||||
ocr_workers: int = 2
|
||||
max_request_bytes: int = 32 * 1024 * 1024
|
||||
abandon_after_hours: int = 24
|
||||
sweep_seconds: int = 60
|
||||
stale_after_seconds: int = 300
|
||||
max_attempts: int = 3
|
||||
# S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen
|
||||
upload_connect_timeout_seconds: int = 5
|
||||
upload_read_timeout_seconds: int = 30
|
||||
upload_max_attempts: int = 2 # inkl. erstem Versuch
|
||||
# Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen
|
||||
alarm_after_seconds: int = 1800
|
||||
|
||||
|
||||
def load_gundi_config(path: str) -> GundiConfig:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
raw = yaml.safe_load(f) or {}
|
||||
g = raw.get("gundi") or {}
|
||||
defaults = GundiConfig(enabled=False, api_key="")
|
||||
|
||||
def prefix(key: str) -> str:
|
||||
value = str(g.get(key, getattr(defaults, key)))
|
||||
return value if value.endswith("/") else value + "/"
|
||||
|
||||
gcfg = GundiConfig(
|
||||
enabled=bool(g.get("enabled", False)),
|
||||
api_key=os.environ.get("GUNDI_API_KEY", ""),
|
||||
bind_host=str(g.get("bind_host", defaults.bind_host)),
|
||||
bind_port=int(g.get("bind_port", defaults.bind_port)),
|
||||
inbox_prefix=prefix("inbox_prefix"),
|
||||
processed_prefix=prefix("processed_prefix"),
|
||||
thumb_prefix=prefix("thumb_prefix"),
|
||||
workers=max(1, int(g.get("workers", defaults.workers))),
|
||||
ocr_workers=max(1, int(g.get("ocr_workers", defaults.ocr_workers))),
|
||||
max_request_bytes=int(g.get("max_request_bytes", defaults.max_request_bytes)),
|
||||
abandon_after_hours=int(g.get("abandon_after_hours", defaults.abandon_after_hours)),
|
||||
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
|
||||
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
|
||||
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
|
||||
upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))),
|
||||
upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))),
|
||||
upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))),
|
||||
alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))),
|
||||
)
|
||||
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
|
||||
raise SystemExit(
|
||||
f"GUNDI_API_KEY fehlt oder ist kuerzer als {MIN_API_KEY_LEN} Zeichen "
|
||||
"(gundi.enabled ist aktiv). Bitte in der .env setzen."
|
||||
)
|
||||
return gcfg
|
||||
|
||||
|
||||
def connect(dsn: str) -> psycopg.Connection:
|
||||
return psycopg.connect(dsn, connect_timeout=10)
|
||||
|
||||
|
||||
def apply_schema(dsn: str) -> None:
|
||||
sql = SCHEMA_FILE.read_text(encoding="utf-8")
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
def utc_now_str() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Validierung + Mapping
|
||||
# -----------------------
|
||||
class EventError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _num(v) -> Optional[float]:
|
||||
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
||||
return None
|
||||
return float(v)
|
||||
|
||||
|
||||
def _int(v) -> Optional[int]:
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
if isinstance(v, int):
|
||||
return v
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str) and re.fullmatch(r"-?\d+", v.strip()):
|
||||
return int(v.strip())
|
||||
return None
|
||||
|
||||
|
||||
def _bool(v) -> Optional[bool]:
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if isinstance(v, str) and v.strip().lower() in ("true", "false"):
|
||||
return v.strip().lower() == "true"
|
||||
return None
|
||||
|
||||
|
||||
def _str(v) -> Optional[str]:
|
||||
if v is None:
|
||||
return None
|
||||
s = str(v).strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def parse_ts(value) -> datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise EventError("recorded_at fehlt")
|
||||
s = value.strip()
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
raise EventError(f"recorded_at ungueltig: {value!r}") from None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def coords(location) -> tuple[Optional[float], Optional[float]]:
|
||||
"""ICU meldet unbekannte Koordinaten als 0.0/0.0 -> NULL."""
|
||||
if not isinstance(location, dict):
|
||||
return None, None
|
||||
lat, lon = _num(location.get("lat")), _num(location.get("lon"))
|
||||
if lat is None or lon is None or (lat == 0 and lon == 0):
|
||||
return None, None
|
||||
return lat, lon
|
||||
|
||||
|
||||
def validate_event(ev) -> dict:
|
||||
if not isinstance(ev, dict):
|
||||
raise EventError("Event muss ein JSON-Objekt sein")
|
||||
event_type = ev.get("event_type")
|
||||
if not isinstance(event_type, str) or not event_type:
|
||||
raise EventError("event_type fehlt")
|
||||
source = ev.get("source")
|
||||
if not isinstance(source, str) or not source.strip():
|
||||
raise EventError("source fehlt")
|
||||
details = ev.get("event_details")
|
||||
if details is None:
|
||||
details = {}
|
||||
if not isinstance(details, dict):
|
||||
raise EventError("event_details muss ein Objekt sein")
|
||||
if event_type in EVENT_CAMERATRAP_TYPES:
|
||||
dets = details.get("detections", [])
|
||||
if not isinstance(dets, list) or not all(isinstance(d, dict) for d in dets):
|
||||
raise EventError("event_details.detections muss eine Liste von Objekten sein")
|
||||
lat, lon = coords(ev.get("location"))
|
||||
return {
|
||||
"event_type": event_type,
|
||||
"source": source.strip(),
|
||||
"recorded_at": parse_ts(ev.get("recorded_at")),
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
|
||||
def camera_status_row(env: dict, object_id: str, raw: dict) -> dict:
|
||||
d = env["details"]
|
||||
return {
|
||||
"imei": env["source"],
|
||||
"reported_at": env["recorded_at"],
|
||||
"iccid": _str(d.get("iccid")),
|
||||
"init_key": _str(d.get("init_key")),
|
||||
"camera_name": _str(d.get("camera_name")),
|
||||
"camera_model": _str(d.get("camera_model")),
|
||||
"firmware_version": _str(d.get("firmware_version")),
|
||||
"camera_language": _str(d.get("camera_language")),
|
||||
"customer_timezone": _str(d.get("customer_timezone")),
|
||||
"mcc": _str(d.get("mobile_country_code")),
|
||||
"mnc": _str(d.get("mobile_network_code")),
|
||||
"lac": _str(d.get("location_area_code")),
|
||||
"cell_id": _str(d.get("cell_id")),
|
||||
"battery_pct": _int(d.get("battery_status")),
|
||||
"signal_strength": _int(d.get("signal_strength")),
|
||||
"temperature_c": _int(d.get("temperature")),
|
||||
"coordinate_switch": _bool(d.get("coordinate_switch")),
|
||||
"work_period": _bool(d.get("work_period")),
|
||||
"work_start": _str(d.get("work_start")),
|
||||
"work_end": _str(d.get("work_end")),
|
||||
"image_format": _str(d.get("image_format")),
|
||||
"image_night": _str(d.get("image_night")),
|
||||
"thumbnail_size": _str(d.get("thumbnail_size")),
|
||||
"pir_interval": _str(d.get("pir_interval")),
|
||||
"time_scan": _str(d.get("time_scan")),
|
||||
"lat": env["lat"],
|
||||
"lon": env["lon"],
|
||||
"source": "gundi",
|
||||
"source_ref": object_id,
|
||||
"raw": Json(raw),
|
||||
}
|
||||
|
||||
|
||||
def detection_rows(env: dict, image_uuid: str) -> list[dict]:
|
||||
rows = []
|
||||
for det in env["details"].get("detections") or []:
|
||||
category = _str(det.get("category"))
|
||||
if category is None:
|
||||
continue
|
||||
pct = _num(det.get("confidence_percent"))
|
||||
confidence = None if pct is None else min(max(pct / 100.0, 0.0), 1.0)
|
||||
bbox = [_int(det.get(k)) for k in ("x_px", "y_px", "width_px", "height_px")]
|
||||
if all(v in (None, 0) for v in bbox):
|
||||
bbox = [None, None, None, None]
|
||||
rows.append({
|
||||
"image_uuid": image_uuid,
|
||||
"category": category,
|
||||
"confidence": confidence,
|
||||
"bbox_x": bbox[0],
|
||||
"bbox_y": bbox[1],
|
||||
"bbox_w": bbox[2],
|
||||
"bbox_h": bbox[3],
|
||||
"model_version": "icu",
|
||||
"source": "gundi",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def image_row(env: dict, object_id: str, raw: dict, size_bytes: int, processed_key: str, thumb_key: str) -> dict:
|
||||
d = env["details"]
|
||||
return {
|
||||
"image_uuid": object_id,
|
||||
"imei": env["source"],
|
||||
"servertime": env["recorded_at"],
|
||||
"ocr_state": "PENDING",
|
||||
"camera_name": _str(d.get("cameratraprep_camera-name")),
|
||||
"camera_make": _str(d.get("cameratraprep_camera-make")),
|
||||
"lat": env["lat"],
|
||||
"lon": env["lon"],
|
||||
"size_bytes": size_bytes,
|
||||
"processed_key": processed_key,
|
||||
"thumbnail_key": thumb_key,
|
||||
"source": "gundi",
|
||||
"source_ref": object_id,
|
||||
"raw": Json(raw),
|
||||
}
|
||||
|
||||
|
||||
def _insert(cur, table: str, row: dict, suffix: str = "") -> int:
|
||||
cols = list(row)
|
||||
cur.execute(
|
||||
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join(['%s'] * len(cols))}) {suffix}",
|
||||
[row[c] for c in cols],
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Receiver-seitige Persistenz (synchron im Request)
|
||||
# -----------------------
|
||||
def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
|
||||
"""Ein Request = eine Transaktion. Antwort erst nach Commit."""
|
||||
results = []
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
for env, raw in items:
|
||||
object_id = str(uuidlib.uuid4())
|
||||
if env["event_type"] == EVENT_STATUS:
|
||||
state = "INGESTED"
|
||||
elif env["event_type"] in EVENT_CAMERATRAP_TYPES:
|
||||
state = "AWAITING_ATTACHMENT"
|
||||
else:
|
||||
state = "IGNORED"
|
||||
_insert(cur, "remote_cam.gundi_inbox", {
|
||||
"object_id": object_id,
|
||||
"event_type": env["event_type"],
|
||||
"source": env["source"],
|
||||
"recorded_at": env["recorded_at"],
|
||||
"payload": Json(raw),
|
||||
"state": state,
|
||||
})
|
||||
if env["event_type"] == EVENT_STATUS:
|
||||
_insert(
|
||||
cur, "remote_cam.camera_status", camera_status_row(env, object_id, raw),
|
||||
"ON CONFLICT (imei, reported_at) WHERE source = 'gundi' DO NOTHING",
|
||||
)
|
||||
results.append({
|
||||
"object_id": object_id,
|
||||
"created_at": utc_now_str(),
|
||||
"event_type": env["event_type"],
|
||||
"source": env["source"],
|
||||
"state": state,
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
|
||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.
|
||||
|
||||
Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung
|
||||
blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro
|
||||
object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt.
|
||||
"""
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s",
|
||||
(object_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row or row[0] not in EVENT_CAMERATRAP_TYPES:
|
||||
raise LookupError(object_id)
|
||||
if row[1] in ("RECEIVED", "INGESTED"):
|
||||
return "duplicate"
|
||||
|
||||
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
||||
s3.put_bytes(key, jpg, "image/jpeg")
|
||||
|
||||
with connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
|
||||
attempts = 0, last_error = NULL, updated_at = now()
|
||||
WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED')
|
||||
""",
|
||||
(key, object_id),
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
return "duplicate"
|
||||
return "stored"
|
||||
|
||||
|
||||
# -----------------------
|
||||
# EXIF
|
||||
# -----------------------
|
||||
def exif_set_digitized(jpg: bytes, servertime: datetime, *, timeout_seconds: int) -> bytes:
|
||||
"""Stufe 1: nur Serverzeit. DateTimeOriginal folgt nach dem OCR (Stufe 2)."""
|
||||
dtd = to_exif_datetime(servertime.astimezone(timezone.utc))
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
path = os.path.join(td, "in.jpg")
|
||||
with open(path, "wb") as f:
|
||||
f.write(jpg)
|
||||
subprocess.run(
|
||||
[
|
||||
"exiftool", "-overwrite_original",
|
||||
f"-DateTimeDigitized={dtd}", "-OffsetTimeDigitized=+00:00",
|
||||
f"-ModifyDate={dtd}", "-OffsetTime=+00:00",
|
||||
path,
|
||||
],
|
||||
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_seconds,
|
||||
)
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Pipeline
|
||||
# -----------------------
|
||||
class Pipeline:
|
||||
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
|
||||
self.cfg = cfg
|
||||
self.gcfg = gcfg
|
||||
# Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell.
|
||||
# Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt).
|
||||
self.s3 = S3(cfg, self._s3_config(
|
||||
cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds,
|
||||
cfg.s3_max_attempts, cfg.s3_retry_mode,
|
||||
))
|
||||
self.s3_request = S3(cfg, self._s3_config(
|
||||
cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds,
|
||||
gcfg.upload_max_attempts, "standard",
|
||||
))
|
||||
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
|
||||
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
|
||||
self._stop = threading.Event()
|
||||
# Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf)
|
||||
self._inflight: set[tuple[str, str]] = set()
|
||||
self._inflight_lock = threading.Lock()
|
||||
self._last_alarm = 0.0
|
||||
|
||||
@staticmethod
|
||||
def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int,
|
||||
max_attempts: int, retry_mode: str) -> BotoClientConfig:
|
||||
return BotoClientConfig(
|
||||
max_pool_connections=cfg.s3_max_pool_connections,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"mode": retry_mode, "total_max_attempts": max_attempts},
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._ingest_pool.shutdown(wait=False, cancel_futures=True)
|
||||
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_ingest(self, object_id: str) -> None:
|
||||
self._submit(self._ingest_pool, self.ingest, object_id)
|
||||
|
||||
def submit_ocr(self, image_uuid: str) -> None:
|
||||
self._submit(self._ocr_pool, self.ocr, image_uuid)
|
||||
|
||||
def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None:
|
||||
token = (fn.__name__, arg)
|
||||
with self._inflight_lock:
|
||||
if token in self._inflight:
|
||||
return
|
||||
self._inflight.add(token)
|
||||
try:
|
||||
pool.submit(self._guarded, token, fn, arg)
|
||||
except Exception:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(token)
|
||||
raise
|
||||
|
||||
def _guarded(self, token: tuple[str, str], fn, arg: str) -> None:
|
||||
try:
|
||||
fn(arg)
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(token)
|
||||
|
||||
# --- Stufe 1 ---------------------------------------------------------
|
||||
def ingest(self, object_id: str) -> None:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT payload, attachment_key FROM remote_cam.gundi_inbox
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
(object_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return
|
||||
payload, attachment_key = row
|
||||
env = validate_event(payload)
|
||||
jpg = self.s3.get_bytes(attachment_key)
|
||||
|
||||
exif_note = "ok"
|
||||
try:
|
||||
processed = exif_set_digitized(
|
||||
jpg, env["recorded_at"], timeout_seconds=self.cfg.exiftool_timeout_seconds
|
||||
)
|
||||
except Exception as e:
|
||||
processed = jpg
|
||||
exif_note = f"fail({type(e).__name__})"
|
||||
thumb = make_thumbnail_bytes(jpg, self.cfg.thumb_max)
|
||||
|
||||
processed_key = f"{self.gcfg.processed_prefix}{object_id}.jpg"
|
||||
thumb_key = f"{self.gcfg.thumb_prefix}{object_id}.jpg"
|
||||
self.s3.put_bytes(processed_key, processed, "image/jpeg")
|
||||
self.s3.put_bytes(thumb_key, thumb, "image/jpeg")
|
||||
|
||||
n_det = 0
|
||||
inserted = _insert(
|
||||
cur, "remote_cam.image",
|
||||
image_row(env, object_id, payload, len(jpg), processed_key, thumb_key),
|
||||
"ON CONFLICT (image_uuid) DO NOTHING",
|
||||
)
|
||||
if inserted:
|
||||
for det in detection_rows(env, object_id):
|
||||
n_det += _insert(cur, "remote_cam.detection", det)
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'INGESTED', last_error = NULL, updated_at = now()
|
||||
WHERE object_id = %s
|
||||
""",
|
||||
(object_id,),
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_ingest_failure(object_id, e)
|
||||
return
|
||||
|
||||
try:
|
||||
self.s3.delete_keys([attachment_key])
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] inbox-cleanup {object_id} fehlgeschlagen: {e}", flush=True)
|
||||
self.submit_ocr(object_id)
|
||||
ms = int((time.perf_counter() - start) * 1000)
|
||||
print(
|
||||
f"[GUNDI] ingest object_id={object_id} imei={env['source']} detections={n_det} "
|
||||
f"exif={exif_note} ms={ms}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
if is_transient_error(err):
|
||||
# S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min
|
||||
# Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder.
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET last_error = %s, updated_at = now()
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
""",
|
||||
(msg, object_id),
|
||||
)
|
||||
print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||
return
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET attempts = attempts + 1,
|
||||
state = CASE WHEN attempts + 1 >= %s THEN 'FAILED' ELSE state END,
|
||||
last_error = %s, updated_at = now()
|
||||
WHERE object_id = %s AND state = 'RECEIVED'
|
||||
RETURNING attempts, state
|
||||
""",
|
||||
(self.gcfg.max_attempts, msg, object_id),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
print(f"[GUNDI] ingest object_id={object_id} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True)
|
||||
|
||||
# --- Stufe 2 ---------------------------------------------------------
|
||||
def ocr(self, image_uuid: str) -> None:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT imei, servertime, processed_key FROM remote_cam.image
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
(image_uuid,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return
|
||||
imei, servertime, processed_key = row
|
||||
tzinfo, tzname = self._camera_tz(cur, imei, servertime)
|
||||
jpg = self.s3.get_bytes(processed_key)
|
||||
ocr_ts = ocr_extract_timestamp(
|
||||
jpg, self.cfg.ocr_crop_w_frac, self.cfg.ocr_crop_h_frac, tzinfo
|
||||
)
|
||||
|
||||
exif_err = None
|
||||
try:
|
||||
patched = exif_write_with_exiftool(
|
||||
jpg, ocr_ts, servertime, timeout_seconds=self.cfg.exiftool_timeout_seconds
|
||||
)
|
||||
self.s3.put_bytes(processed_key, patched, "image/jpeg")
|
||||
except Exception as e:
|
||||
exif_err = f"exif: {type(e).__name__}: {e}"[:1000]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_ts = %s, ocr_state = 'OK', ocr_attempts = ocr_attempts + 1,
|
||||
ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s
|
||||
""",
|
||||
(ocr_ts, exif_err, image_uuid),
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_ocr_failure(image_uuid, e)
|
||||
return
|
||||
|
||||
drift = int((servertime - ocr_ts).total_seconds())
|
||||
ms = int((time.perf_counter() - start) * 1000)
|
||||
print(
|
||||
f"[GUNDI] ocr image_uuid={image_uuid} ocr_ts={ocr_ts.isoformat()} tz={tzname} "
|
||||
f"servertime-ocr={drift}s exif={'ok' if exif_err is None else 'fail'} ms={ms}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _camera_tz(cur, imei: str, servertime: datetime):
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT customer_timezone FROM remote_cam.camera_status
|
||||
WHERE imei = %s AND customer_timezone IS NOT NULL AND reported_at <= %s
|
||||
ORDER BY reported_at DESC LIMIT 1
|
||||
""",
|
||||
(imei, servertime),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT customer_timezone FROM remote_cam.camera_status
|
||||
WHERE imei = %s AND customer_timezone IS NOT NULL
|
||||
ORDER BY reported_at ASC LIMIT 1
|
||||
""",
|
||||
(imei,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
name = row[0] if row else "UTC"
|
||||
zone = tz.gettz(name)
|
||||
if zone is None:
|
||||
return timezone.utc, "UTC"
|
||||
return zone, name
|
||||
|
||||
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
|
||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||
if is_transient_error(err):
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
""",
|
||||
(msg, image_uuid),
|
||||
)
|
||||
print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||
return
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_attempts = ocr_attempts + 1,
|
||||
ocr_state = CASE WHEN ocr_attempts + 1 >= %s THEN 'FAIL' ELSE ocr_state END,
|
||||
ocr_error = %s, updated_at = now()
|
||||
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||
RETURNING ocr_attempts, ocr_state
|
||||
""",
|
||||
(self.gcfg.max_attempts, msg, image_uuid),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
print(f"[GUNDI] ocr image_uuid={image_uuid} FEHLER attempt={row[0]} state={row[1]} error={msg}", flush=True)
|
||||
|
||||
# --- Sweep -----------------------------------------------------------
|
||||
def _sweep_loop(self) -> None:
|
||||
delay = 0
|
||||
while not self._stop.wait(delay):
|
||||
delay = self.gcfg.sweep_seconds
|
||||
try:
|
||||
self.sweep()
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] sweep Fehler: {type(e).__name__}: {e}", flush=True)
|
||||
|
||||
def sweep(self) -> None:
|
||||
stale = self.gcfg.stale_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT object_id::text FROM remote_cam.gundi_inbox
|
||||
WHERE state = 'RECEIVED' AND updated_at < now() - make_interval(secs => %s)
|
||||
ORDER BY updated_at LIMIT 200
|
||||
""",
|
||||
(stale,),
|
||||
)
|
||||
ingest_ids = [r[0] for r in cur.fetchall()]
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT image_uuid::text FROM remote_cam.image
|
||||
WHERE ocr_state = 'PENDING' AND source = 'gundi'
|
||||
AND updated_at < now() - make_interval(secs => %s)
|
||||
ORDER BY updated_at LIMIT 200
|
||||
""",
|
||||
(stale,),
|
||||
)
|
||||
ocr_ids = [r[0] for r in cur.fetchall()]
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'ABANDONED', updated_at = now()
|
||||
WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(hours => %s)
|
||||
""",
|
||||
(self.gcfg.abandon_after_hours,),
|
||||
)
|
||||
abandoned = cur.rowcount
|
||||
for object_id in ingest_ids:
|
||||
self.submit_ingest(object_id)
|
||||
for image_uuid in ocr_ids:
|
||||
self.submit_ocr(image_uuid)
|
||||
if ingest_ids or ocr_ids or abandoned:
|
||||
print(
|
||||
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
|
||||
flush=True,
|
||||
)
|
||||
self.check_backlog()
|
||||
|
||||
def check_backlog(self) -> None:
|
||||
"""Meldet haengende Eintraege und NEUE Fehlschlaege (letzte 24 h) als [GUNDI][ALARM], hoechstens alle 5 min.
|
||||
|
||||
Aeltere FAILED/FAIL bleiben in der DB (gundi_requeue), loesen aber keinen Dauer-Alarm aus.
|
||||
"""
|
||||
limit = self.gcfg.alarm_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE state = 'RECEIVED'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'FAILED' AND updated_at > now() - interval '24 hours')
|
||||
FROM remote_cam.gundi_inbox
|
||||
WHERE received_at > now() - interval '14 days'
|
||||
""",
|
||||
(limit, limit),
|
||||
)
|
||||
received_stuck, awaiting_old, failed = cur.fetchone()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE ocr_state = 'PENDING'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours')
|
||||
FROM remote_cam.image
|
||||
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
ocr_stuck, ocr_failed = cur.fetchone()
|
||||
if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed):
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now - self._last_alarm < 300:
|
||||
return
|
||||
self._last_alarm = now
|
||||
print(
|
||||
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
|
||||
f"failed_24h={failed} ocr_haengt={ocr_stuck} ocr_fail_24h={ocr_failed} (haengt = laenger als {limit}s) "
|
||||
"- Details/Zuruecksetzen: tools/gundi_requeue.py",
|
||||
flush=True,
|
||||
)
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gundi-v2-kompatibler Empfaenger fuer ICU-Push-Events.
|
||||
|
||||
POST /v2/events/ -> {"object_id", "created_at"}
|
||||
POST /v2/events/<object_id>/attachments/ -> {"object_id", "created_at"}
|
||||
GET /healthz
|
||||
"""
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from email.parser import BytesParser
|
||||
from email.policy import default as email_policy
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from gundi_pipeline import (
|
||||
UUID_STRICT_RE,
|
||||
EventError,
|
||||
GundiConfig,
|
||||
Pipeline,
|
||||
apply_schema,
|
||||
attach_image,
|
||||
load_gundi_config,
|
||||
store_events,
|
||||
utc_now_str,
|
||||
validate_event,
|
||||
)
|
||||
from main import AppConfig, is_transient_error, load_config
|
||||
|
||||
EVENTS_PATH = "/v2/events/"
|
||||
RETRY_AFTER_SECONDS = 30
|
||||
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
|
||||
def extract_jpeg(content_type: str, body: bytes) -> tuple[bytes | None, int]:
|
||||
"""Erstes JPEG aus einem multipart/form-data-Body; Client-Dateinamen werden ignoriert."""
|
||||
raw = f"Content-Type: {content_type}\r\nMIME-Version: 1.0\r\n\r\n".encode("latin-1") + body
|
||||
msg = BytesParser(policy=email_policy).parsebytes(raw)
|
||||
if not msg.is_multipart():
|
||||
return None, 0
|
||||
files = []
|
||||
for part in msg.iter_parts():
|
||||
if part.get_content_disposition() != "form-data" or not part.get_filename():
|
||||
continue
|
||||
files.append(part.get_payload(decode=True) or b"")
|
||||
jpgs = [f for f in files if f.startswith(JPEG_MAGIC)]
|
||||
return (jpgs[0] if jpgs else None), len(files)
|
||||
|
||||
|
||||
class GundiServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, addr, cfg: AppConfig, gcfg: GundiConfig, pipeline: Pipeline):
|
||||
super().__init__(addr, GundiHandler)
|
||||
self.cfg = cfg
|
||||
self.gcfg = gcfg
|
||||
self.pipeline = pipeline
|
||||
|
||||
|
||||
class GundiHandler(BaseHTTPRequestHandler):
|
||||
server: GundiServer
|
||||
server_version = "melesICUmover-gundi/1.0"
|
||||
sys_version = ""
|
||||
timeout = 60
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
client = self.headers.get("X-Real-IP") if self.headers else None
|
||||
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
|
||||
|
||||
# --- Antworten -------------------------------------------------------
|
||||
def _json(self, code: int, data, *, retry_after: int | None = None) -> None:
|
||||
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
if retry_after is not None:
|
||||
self.send_header("Retry-After", str(retry_after))
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None:
|
||||
if close:
|
||||
self.close_connection = True
|
||||
self._json(code, {"error": message}, retry_after=retry_after)
|
||||
|
||||
def _redirect(self, location: str) -> None:
|
||||
# 308 statt 301: der Client muss POST + Body beibehalten.
|
||||
self.close_connection = True
|
||||
self.send_response(308)
|
||||
self.send_header("Location", location)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# --- Request-Helfer --------------------------------------------------
|
||||
def _authorized(self) -> bool:
|
||||
got = (self.headers.get("apikey") or "").encode("utf-8")
|
||||
return hmac.compare_digest(got, self.server.gcfg.api_key.encode("utf-8"))
|
||||
|
||||
def _read_body(self) -> bytes | None:
|
||||
if self.headers.get("Transfer-Encoding"):
|
||||
self._error(411, "Transfer-Encoding wird nicht unterstuetzt, Content-Length senden", close=True)
|
||||
return None
|
||||
raw_len = self.headers.get("Content-Length")
|
||||
if raw_len is None:
|
||||
self._error(411, "Content-Length erforderlich", close=True)
|
||||
return None
|
||||
try:
|
||||
length = int(raw_len)
|
||||
except ValueError:
|
||||
self._error(400, "Content-Length ungueltig", close=True)
|
||||
return None
|
||||
if length < 0:
|
||||
self._error(400, "Content-Length ungueltig", close=True)
|
||||
return None
|
||||
if length > self.server.gcfg.max_request_bytes:
|
||||
self._error(413, "Request zu gross", close=True)
|
||||
return None
|
||||
body = self.rfile.read(length)
|
||||
if len(body) != length:
|
||||
self._error(400, "Body unvollstaendig", close=True)
|
||||
return None
|
||||
return body
|
||||
|
||||
# --- Routing ---------------------------------------------------------
|
||||
def do_GET(self) -> None:
|
||||
if urlparse(self.path).path == "/healthz":
|
||||
self._json(200, {"status": "ok"})
|
||||
else:
|
||||
self._error(404, "not found")
|
||||
|
||||
def do_POST(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if path == "/v2/events":
|
||||
self._redirect(EVENTS_PATH)
|
||||
return
|
||||
attachment = ATTACHMENT_RE.match(path)
|
||||
if attachment and not attachment.group(2):
|
||||
self._redirect(path + "/")
|
||||
return
|
||||
if path != EVENTS_PATH and not attachment:
|
||||
self._error(404, "not found", close=True)
|
||||
return
|
||||
if not self._authorized():
|
||||
self._error(401, "Invalid or missing apikey", close=True)
|
||||
return
|
||||
body = self._read_body()
|
||||
if body is None:
|
||||
return
|
||||
try:
|
||||
if attachment:
|
||||
self._handle_attachment(attachment.group(1).lower(), body)
|
||||
else:
|
||||
self._handle_events(body)
|
||||
except Exception as e:
|
||||
if is_transient_error(e):
|
||||
# S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt
|
||||
print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS)
|
||||
else:
|
||||
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(500, "internal error", close=True)
|
||||
|
||||
# --- Handler ---------------------------------------------------------
|
||||
def _handle_events(self, body: bytes) -> None:
|
||||
if "application/json" not in (self.headers.get("Content-Type") or "").lower():
|
||||
self._error(415, "Content-Type muss application/json sein")
|
||||
return
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
self._error(400, "Ungueltiges JSON")
|
||||
return
|
||||
events = payload if isinstance(payload, list) else [payload]
|
||||
if not events:
|
||||
self._error(400, "Leere Event-Liste")
|
||||
return
|
||||
try:
|
||||
envs = [validate_event(ev) for ev in events]
|
||||
except EventError as e:
|
||||
self._error(400, str(e))
|
||||
return
|
||||
|
||||
results = store_events(self.server.cfg.pg_dsn, list(zip(envs, events)))
|
||||
for r in results:
|
||||
print(
|
||||
f"[GUNDI] event object_id={r['object_id']} type={r['event_type']} "
|
||||
f"imei={r['source']} state={r['state']}",
|
||||
flush=True,
|
||||
)
|
||||
out = [{"object_id": r["object_id"], "created_at": r["created_at"]} for r in results]
|
||||
self._json(200, out if isinstance(payload, list) else out[0])
|
||||
|
||||
def _handle_attachment(self, object_id: str, body: bytes) -> None:
|
||||
if not UUID_STRICT_RE.match(object_id):
|
||||
self._error(404, "Unknown object_id")
|
||||
return
|
||||
content_type = self.headers.get("Content-Type") or ""
|
||||
if "multipart/form-data" not in content_type.lower():
|
||||
self._error(415, "Content-Type muss multipart/form-data sein")
|
||||
return
|
||||
jpg, n_files = extract_jpeg(content_type, body)
|
||||
if jpg is None:
|
||||
self._error(400, "Kein JPEG-Anhang gefunden")
|
||||
return
|
||||
try:
|
||||
outcome = attach_image(
|
||||
self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg
|
||||
)
|
||||
except LookupError:
|
||||
self._error(404, "Unknown object_id")
|
||||
return
|
||||
if outcome == "stored":
|
||||
self.server.pipeline.submit_ingest(object_id)
|
||||
print(
|
||||
f"[GUNDI] attachment object_id={object_id} bytes={len(jpg)} files={n_files} outcome={outcome}",
|
||||
flush=True,
|
||||
)
|
||||
self._json(200, {"object_id": object_id, "created_at": utc_now_str()})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
|
||||
cfg = load_config(cfg_path)
|
||||
gcfg = load_gundi_config(cfg_path)
|
||||
if not gcfg.enabled:
|
||||
print("[GUNDI] gundi.enabled ist false - Receiver bleibt inaktiv.", flush=True)
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
|
||||
apply_schema(cfg.pg_dsn)
|
||||
pipeline = Pipeline(cfg, gcfg)
|
||||
pipeline.start()
|
||||
server = GundiServer((gcfg.bind_host, gcfg.bind_port), cfg, gcfg, pipeline)
|
||||
|
||||
def _shutdown(signum, frame):
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, _shutdown)
|
||||
signal.signal(signal.SIGINT, _shutdown)
|
||||
|
||||
print(
|
||||
f"[GUNDI] lausche auf {gcfg.bind_host}:{gcfg.bind_port} "
|
||||
f"inbox={gcfg.inbox_prefix} processed={gcfg.processed_prefix} thumbs={gcfg.thumb_prefix}",
|
||||
flush=True,
|
||||
)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
pipeline.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -6,6 +6,8 @@ import json
|
||||
import time
|
||||
import tempfile
|
||||
import subprocess
|
||||
import threading
|
||||
import traceback
|
||||
import uuid as uuidlib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
@@ -17,6 +19,12 @@ from urllib import request as urlrequest
|
||||
import boto3
|
||||
import psycopg
|
||||
from botocore.config import Config as BotoClientConfig
|
||||
from botocore.exceptions import (
|
||||
ClientError,
|
||||
ConnectionError as BotoConnectionError,
|
||||
HTTPClientError,
|
||||
IncompleteReadError,
|
||||
)
|
||||
from dateutil import tz
|
||||
from PIL import Image, ImageOps, ImageEnhance
|
||||
import pytesseract
|
||||
@@ -25,14 +33,6 @@ import pytesseract
|
||||
import yaml
|
||||
|
||||
|
||||
if os.environ.get("DEBUG") == "1":
|
||||
import debugpy
|
||||
debugpy.listen(("0.0.0.0", 5678))
|
||||
print("Waiting for debugger...")
|
||||
debugpy.wait_for_client()
|
||||
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Config
|
||||
# -----------------------
|
||||
@@ -81,6 +81,17 @@ class AppConfig:
|
||||
deermapper_api_timeout_seconds: int = 20
|
||||
deermapper_api_image_field: str = "image"
|
||||
|
||||
# Robustheit Poller: S3-Timeouts/Retries, DB-Connect-Timeout, Backoff, Lebenszeichen
|
||||
s3_connect_timeout_seconds: float = 10.0
|
||||
s3_read_timeout_seconds: float = 30.0
|
||||
s3_retry_mode: str = "standard"
|
||||
s3_max_attempts: int = 4
|
||||
pg_connect_timeout_seconds: int = 10
|
||||
backoff_initial_seconds: float = 5.0
|
||||
backoff_max_seconds: float = 300.0
|
||||
heartbeat_seconds: int = 300
|
||||
heartbeat_file: str = "/tmp/melesicumover.heartbeat"
|
||||
|
||||
|
||||
def load_config(path: str) -> AppConfig:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
@@ -101,22 +112,22 @@ def load_config(path: str) -> AppConfig:
|
||||
return value.strip().lower() in ("1", "true", "yes", "on")
|
||||
return bool(value)
|
||||
|
||||
s3 = raw["s3"]
|
||||
pg = raw["postgres"]
|
||||
app = raw.get("app", {})
|
||||
deermapper = raw.get("deermapper-api", raw.get("deermapper_api", {}))
|
||||
s3 = raw.get("s3") or {}
|
||||
pg = raw.get("postgres") or {}
|
||||
app = raw.get("app") or {}
|
||||
deermapper = raw.get("deermapper-api") or raw.get("deermapper_api") or {}
|
||||
default_workers = max(1, min(16, (os.cpu_count() or 2) * 2))
|
||||
parallel_workers = int(app.get("parallel_workers", default_workers))
|
||||
parallel_workers = max(1, parallel_workers)
|
||||
s3_pool = int(app.get("s3_max_pool_connections", max(16, parallel_workers * 4)))
|
||||
s3_pool = max(10, s3_pool)
|
||||
|
||||
return AppConfig(
|
||||
s3_endpoint=env_or("S3_ENDPOINT", s3["endpoint"]),
|
||||
s3_access_key=env_or("S3_ACCESS_KEY", s3["access_key"]),
|
||||
s3_secret_key=env_or("S3_SECRET_KEY", s3["secret_key"]),
|
||||
s3_bucket=env_or("S3_BUCKET", s3["bucket"]),
|
||||
pg_dsn=env_or("PG_DSN", pg["dsn"]),
|
||||
cfg = AppConfig(
|
||||
s3_endpoint=env_or("S3_ENDPOINT", s3.get("endpoint", "")),
|
||||
s3_access_key=env_or("S3_ACCESS_KEY", s3.get("access_key", "")),
|
||||
s3_secret_key=env_or("S3_SECRET_KEY", s3.get("secret_key", "")),
|
||||
s3_bucket=env_or("S3_BUCKET", s3.get("bucket", "")),
|
||||
pg_dsn=env_or("PG_DSN", pg.get("dsn", "")),
|
||||
entrance_prefix=app.get("entrance_prefix", "icu/entrance/"),
|
||||
processed_prefix=app.get("processed_prefix", "icu/processed/"),
|
||||
thumb_prefix=app.get("thumb_prefix", "icu/thumbnails/"),
|
||||
@@ -136,8 +147,40 @@ def load_config(path: str) -> AppConfig:
|
||||
deermapper_api_key=env_or("DEERMAPPER_API_KEY", deermapper.get("apiKey", "")),
|
||||
deermapper_api_timeout_seconds=int(app.get("deermapper_api_timeout_seconds", 20)),
|
||||
deermapper_api_image_field=app.get("deermapper_api_image_field", "image"),
|
||||
s3_connect_timeout_seconds=float(app.get("s3_connect_timeout_seconds", 10)),
|
||||
s3_read_timeout_seconds=float(app.get("s3_read_timeout_seconds", 30)),
|
||||
s3_retry_mode=str(app.get("s3_retry_mode", "standard")),
|
||||
s3_max_attempts=max(1, int(app.get("s3_max_attempts", 4))),
|
||||
pg_connect_timeout_seconds=max(1, int(app.get("pg_connect_timeout_seconds", 10))),
|
||||
backoff_initial_seconds=max(0.1, float(app.get("backoff_initial_seconds", 5))),
|
||||
backoff_max_seconds=max(1.0, float(app.get("backoff_max_seconds", 300))),
|
||||
heartbeat_seconds=max(10, int(app.get("heartbeat_seconds", 300))),
|
||||
heartbeat_file=app.get("heartbeat_file", "/tmp/melesicumover.heartbeat"),
|
||||
)
|
||||
|
||||
# Fail-fast: Secrets kommen ausschliesslich aus der Umgebung (.env). Fehlen sie,
|
||||
# bricht der Start mit klarer Meldung ab, statt spaeter kryptisch zu scheitern.
|
||||
missing = []
|
||||
if not cfg.s3_endpoint:
|
||||
missing.append("S3_ENDPOINT (oder s3.endpoint)")
|
||||
if not cfg.s3_access_key:
|
||||
missing.append("S3_ACCESS_KEY")
|
||||
if not cfg.s3_secret_key:
|
||||
missing.append("S3_SECRET_KEY")
|
||||
if not cfg.s3_bucket:
|
||||
missing.append("S3_BUCKET (oder s3.bucket)")
|
||||
if not cfg.pg_dsn:
|
||||
missing.append("PG_DSN")
|
||||
if cfg.enable_deermapper_api and not cfg.deermapper_api_key:
|
||||
missing.append("DEERMAPPER_API_KEY (enable_deermapper_api ist aktiv)")
|
||||
if missing:
|
||||
raise SystemExit(
|
||||
"Fehlende Konfiguration/Secrets: " + ", ".join(missing)
|
||||
+ ". Bitte in der .env / als Umgebungsvariablen setzen."
|
||||
)
|
||||
|
||||
return cfg
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Helpers
|
||||
@@ -200,13 +243,18 @@ def ocr_extract_timestamp(jpg_bytes: bytes, crop_w_frac: float, crop_h_frac: flo
|
||||
|
||||
bw = gray.point(lambda p: 255 if p > 180 else 0)
|
||||
|
||||
cfg = r'--oem 3 --psm 6 -c tessedit_char_whitelist=0123456789:- '
|
||||
text = pytesseract.image_to_string(bw, config=cfg).strip()
|
||||
tess_cfg = r'--oem 3 --psm 6 -c tessedit_char_whitelist=0123456789:- '
|
||||
text = pytesseract.image_to_string(bw, config=tess_cfg).strip()
|
||||
|
||||
# retry without thresholding
|
||||
text2 = pytesseract.image_to_string(gray, config=cfg).strip()
|
||||
|
||||
dt_local_naive = _parse_ocr_datetime(text, text2)
|
||||
try:
|
||||
# Schnellpfad: der Schwellwert-Durchlauf reicht meist -> zweiten (teuren)
|
||||
# OCR-Lauf einsparen.
|
||||
dt_local_naive = _parse_ocr_datetime(text)
|
||||
except ValueError:
|
||||
# Fallback: ohne Thresholding erneut lesen und beide Durchlaeufe parsen
|
||||
# (identisches Verhalten wie zuvor).
|
||||
text2 = pytesseract.image_to_string(gray, config=tess_cfg).strip()
|
||||
dt_local_naive = _parse_ocr_datetime(text, text2)
|
||||
|
||||
# Keep camera-local timezone (no UTC conversion requested)
|
||||
return dt_local_naive.replace(tzinfo=customer_tzinfo)
|
||||
@@ -383,28 +431,37 @@ def db_init(cur):
|
||||
updated_ts timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
""")
|
||||
cur.execute("""
|
||||
ALTER TABLE remote_cam.import_job
|
||||
ADD COLUMN IF NOT EXISTS needs_ocr_backfill boolean NOT NULL DEFAULT false;
|
||||
""")
|
||||
cur.execute("""
|
||||
ALTER TABLE remote_cam.import_job
|
||||
ADD COLUMN IF NOT EXISTS needs_exif_backfill boolean NOT NULL DEFAULT false;
|
||||
""")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS import_job_needs_ocr_idx ON remote_cam.import_job(needs_ocr_backfill);")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS import_job_needs_exif_idx ON remote_cam.import_job(needs_exif_backfill);")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS import_job_status_idx ON remote_cam.import_job(status);")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS import_job_updated_idx ON remote_cam.import_job(updated_ts);")
|
||||
for col, typedef in [
|
||||
("needs_ocr_backfill", "boolean NOT NULL DEFAULT false"),
|
||||
("needs_exif_backfill", "boolean NOT NULL DEFAULT false"),
|
||||
]:
|
||||
cur.execute("""
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='remote_cam' AND table_name='import_job' AND column_name=%s
|
||||
""", (col,))
|
||||
if not cur.fetchone():
|
||||
cur.execute(f"ALTER TABLE remote_cam.import_job ADD COLUMN {col} {typedef};")
|
||||
for idx, col in [
|
||||
("import_job_needs_ocr_idx", "needs_ocr_backfill"),
|
||||
("import_job_needs_exif_idx", "needs_exif_backfill"),
|
||||
("import_job_status_idx", "status"),
|
||||
("import_job_updated_idx", "updated_ts"),
|
||||
]:
|
||||
cur.execute("SELECT 1 FROM pg_indexes WHERE schemaname='remote_cam' AND indexname=%s", (idx,))
|
||||
if not cur.fetchone():
|
||||
cur.execute(f"CREATE INDEX {idx} ON remote_cam.import_job({col});")
|
||||
|
||||
cur.execute("""
|
||||
CREATE OR REPLACE FUNCTION remote_cam.set_updated_ts()
|
||||
RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.updated_ts = now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
cur.execute("SELECT 1 FROM pg_proc JOIN pg_namespace ON pg_proc.pronamespace=pg_namespace.oid WHERE nspname='remote_cam' AND proname='set_updated_ts'")
|
||||
if not cur.fetchone():
|
||||
cur.execute("""
|
||||
CREATE FUNCTION remote_cam.set_updated_ts()
|
||||
RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.updated_ts = now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
cur.execute("""
|
||||
DO $$
|
||||
BEGIN
|
||||
@@ -425,6 +482,26 @@ def db_init(cur):
|
||||
""")
|
||||
|
||||
|
||||
def pg_connect(cfg: AppConfig) -> psycopg.Connection:
|
||||
# connect_timeout + TCP-Keepalives: ein weggebrochener DB-Host blockiert den
|
||||
# Poller nicht unbegrenzt, sondern endet als OperationalError (-> Backoff).
|
||||
return psycopg.connect(
|
||||
cfg.pg_dsn,
|
||||
connect_timeout=cfg.pg_connect_timeout_seconds,
|
||||
keepalives=1,
|
||||
keepalives_idle=60,
|
||||
keepalives_interval=10,
|
||||
keepalives_count=3,
|
||||
)
|
||||
|
||||
|
||||
def init_db_schema(cfg: AppConfig):
|
||||
with pg_connect(cfg) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
|
||||
def job_upsert_new(cur, uuid: str, jpg_key: str, meta_key: str, cat_key: Optional[str]):
|
||||
cur.execute("""
|
||||
INSERT INTO remote_cam.import_job (image_uuid, status, entrance_jpg_key, entrance_meta_key, entrance_cat_key, has_categories)
|
||||
@@ -647,15 +724,34 @@ def insert_api_response(cur, uuid: str, success: bool, message: str):
|
||||
# -----------------------
|
||||
# S3 ops wrapper
|
||||
# -----------------------
|
||||
def s3_client_config(cfg: AppConfig) -> BotoClientConfig:
|
||||
"""Explizite Timeouts/Retries fuer den Poller (botocore-Default: legacy, 60 s Read-Timeout).
|
||||
|
||||
Pfad-Adressierung + s3v4 wie im Portal; die Adressierung entspricht dem bisherigen
|
||||
Verhalten mit eigenem endpoint_url.
|
||||
"""
|
||||
return BotoClientConfig(
|
||||
max_pool_connections=cfg.s3_max_pool_connections,
|
||||
connect_timeout=cfg.s3_connect_timeout_seconds,
|
||||
read_timeout=cfg.s3_read_timeout_seconds,
|
||||
# total_max_attempts = Versuche inkl. erstem (botocores "max_attempts" zaehlt nur Wiederholungen)
|
||||
retries={"mode": cfg.s3_retry_mode, "total_max_attempts": cfg.s3_max_attempts},
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
)
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self, cfg: AppConfig):
|
||||
def __init__(self, cfg: AppConfig, client_config: Optional[BotoClientConfig] = None):
|
||||
self.cfg = cfg
|
||||
# Ohne client_config bleibt die bisherige Client-Konfiguration (so nutzt sie der
|
||||
# Gundi-Pfad); der Poller uebergibt s3_client_config(cfg).
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=cfg.s3_endpoint,
|
||||
aws_access_key_id=cfg.s3_access_key,
|
||||
aws_secret_access_key=cfg.s3_secret_key,
|
||||
config=BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
|
||||
config=client_config or BotoClientConfig(max_pool_connections=cfg.s3_max_pool_connections),
|
||||
)
|
||||
|
||||
def list_entrance_objects(self):
|
||||
@@ -904,15 +1000,15 @@ def cleanup_entrance_set(s3: S3, s: EntranceSet) -> int:
|
||||
return s3.delete_keys(to_delete)
|
||||
|
||||
|
||||
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
s3c = S3(cfg)
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet], s3c: S3, stats: Optional["PollerStats"] = None):
|
||||
with pg_connect(cfg) as db:
|
||||
for s in batch:
|
||||
start = time.perf_counter()
|
||||
result = "failure"
|
||||
final_status = "UNKNOWN"
|
||||
err_text = None
|
||||
cleanup_deleted = None
|
||||
abort = None
|
||||
try:
|
||||
skip_duplicate = False
|
||||
with db.transaction():
|
||||
@@ -934,6 +1030,11 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
job_set_status(cur, s.uuid, "ERROR", err=str(e))
|
||||
final_status = "ERROR"
|
||||
err_text = str(e)
|
||||
# S3/DB gestoert: Bild bleibt wie bisher auf ERROR, der Rest des Batches wird
|
||||
# nicht mehr angefasst (naechster Zyklus nach Backoff) statt jedes Bild
|
||||
# einzeln gegen die Stoerung laufen zu lassen.
|
||||
if is_transient_error(e):
|
||||
abort = e
|
||||
elapsed_ms = int((time.perf_counter() - start) * 1000)
|
||||
if err_text:
|
||||
print(
|
||||
@@ -947,19 +1048,25 @@ def _process_ready_batch(cfg: AppConfig, batch: list[EntranceSet]):
|
||||
)
|
||||
else:
|
||||
print(f"[RESULT] uuid={s.uuid} result={result} status={final_status} process_ms={elapsed_ms}")
|
||||
if stats is not None:
|
||||
stats.image_done(result == "success")
|
||||
if abort is not None:
|
||||
raise abort
|
||||
if cfg.job_sleep_seconds > 0:
|
||||
time.sleep(cfg.job_sleep_seconds)
|
||||
|
||||
|
||||
def run_once(cfg: AppConfig) -> int:
|
||||
s3c = S3(cfg)
|
||||
def run_once(cfg: AppConfig, stats: Optional["PollerStats"] = None, s3c: Optional[S3] = None) -> int:
|
||||
# Ein Client pro Zyklus fuer alle Worker: boto3-Clients sind threadsicher, das
|
||||
# parallele Anlegen ueber die Default-Session dagegen nicht.
|
||||
s3c = s3c or S3(cfg, s3_client_config(cfg))
|
||||
ready = collect_ready_sets(s3c, cfg)
|
||||
if not ready:
|
||||
return 0
|
||||
|
||||
workers = max(1, min(cfg.parallel_workers, len(ready)))
|
||||
if workers == 1:
|
||||
_process_ready_batch(cfg, ready)
|
||||
_process_ready_batch(cfg, ready, s3c, stats)
|
||||
return len(ready)
|
||||
|
||||
batches: list[list[EntranceSet]] = [[] for _ in range(workers)]
|
||||
@@ -967,26 +1074,254 @@ def run_once(cfg: AppConfig) -> int:
|
||||
batches[i % workers].append(s)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futures = [pool.submit(_process_ready_batch, cfg, batch) for batch in batches if batch]
|
||||
futures = [pool.submit(_process_ready_batch, cfg, batch, s3c, stats) for batch in batches if batch]
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
|
||||
return len(ready)
|
||||
|
||||
|
||||
# -----------------------
|
||||
# Robustheit: Fehlerklassen, Backoff, Lebenszeichen
|
||||
# -----------------------
|
||||
TRANSIENT_S3_CODES = {
|
||||
"ServiceUnavailable", "SlowDown", "Throttling", "ThrottlingException", "TooManyRequests",
|
||||
"RequestLimitExceeded", "RequestTimeout", "InternalError", "500", "502", "503", "504",
|
||||
}
|
||||
|
||||
|
||||
def is_transient_error(exc: Optional[BaseException]) -> bool:
|
||||
"""True fuer Stoerungen, die ohne Codeaenderung vorbeigehen: S3 5xx/Throttling,
|
||||
Verbindungs-/Timeout-Fehler, DB weg (psycopg.OperationalError). Alles andere
|
||||
(4xx wie AccessDenied, Programmierfehler, SQL-Fehler) gilt als unerwartet.
|
||||
Folgt der expliziten Ursache (raise ... from ...)."""
|
||||
seen = set()
|
||||
while exc is not None and id(exc) not in seen:
|
||||
seen.add(id(exc))
|
||||
if isinstance(exc, ClientError):
|
||||
err = exc.response.get("Error") or {}
|
||||
http = (exc.response.get("ResponseMetadata") or {}).get("HTTPStatusCode") or 0
|
||||
if str(err.get("Code", "")) in TRANSIENT_S3_CODES or http >= 500 or http == 429:
|
||||
return True
|
||||
elif isinstance(exc, (BotoConnectionError, HTTPClientError, IncompleteReadError)):
|
||||
return True
|
||||
elif isinstance(exc, psycopg.OperationalError):
|
||||
return True
|
||||
elif isinstance(exc, (ConnectionError, TimeoutError)):
|
||||
return True
|
||||
exc = exc.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def describe_error(exc: BaseException, limit: int = 300) -> str:
|
||||
text = " ".join(str(exc).split())
|
||||
if len(text) > limit:
|
||||
text = text[:limit] + "..."
|
||||
return f"{type(exc).__name__}: {text}"
|
||||
|
||||
|
||||
class Backoff:
|
||||
"""Exponentiell: initial, 2x, 4x ... bis maximum; reset() nach einem erfolgreichen Zyklus."""
|
||||
|
||||
def __init__(self, initial: float, maximum: float, factor: float = 2.0):
|
||||
self.initial = initial
|
||||
self.maximum = maximum
|
||||
self.factor = factor
|
||||
self.failures = 0
|
||||
|
||||
def next_delay(self) -> float:
|
||||
# Exponent begrenzen: bei tagelanger Stoerung sonst OverflowError.
|
||||
delay = min(self.maximum, self.initial * (self.factor ** min(self.failures, 32)))
|
||||
self.failures += 1
|
||||
return delay
|
||||
|
||||
def reset(self):
|
||||
self.failures = 0
|
||||
|
||||
|
||||
def _utc_iso(ts: Optional[float]) -> str:
|
||||
if ts is None:
|
||||
return "never"
|
||||
return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
class PollerStats:
|
||||
"""Zaehler fuer Lebenszeichen-Log und Healthcheck. Threadsicher, weil die Worker je Bild melden.
|
||||
|
||||
Die Heartbeat-Datei wird bei jedem Zyklus und jedem Bild neu geschrieben; ihr mtime
|
||||
zeigt dem Docker-Healthcheck, dass die Schleife vorankommt (wartet vs. haengt)."""
|
||||
|
||||
def __init__(self, heartbeat_file: Optional[str], clock=time.time):
|
||||
self._lock = threading.Lock()
|
||||
self._clock = clock
|
||||
self.heartbeat_file = heartbeat_file
|
||||
self.started = clock()
|
||||
self.cycles = 0
|
||||
self.cycles_failed = 0
|
||||
self.images_ok = 0
|
||||
self.images_failed = 0
|
||||
self.errors_transient = 0
|
||||
self.errors_other = 0
|
||||
self.consecutive_failures = 0
|
||||
self.last_success: Optional[float] = None
|
||||
self.last_error: Optional[str] = None
|
||||
|
||||
def image_done(self, ok: bool):
|
||||
with self._lock:
|
||||
if ok:
|
||||
self.images_ok += 1
|
||||
else:
|
||||
self.images_failed += 1
|
||||
self._write_heartbeat()
|
||||
|
||||
def cycle_ok(self):
|
||||
with self._lock:
|
||||
self.cycles += 1
|
||||
self.consecutive_failures = 0
|
||||
self.last_success = self._clock()
|
||||
self._write_heartbeat()
|
||||
|
||||
def cycle_failed(self, transient: bool, message: str):
|
||||
with self._lock:
|
||||
self.cycles += 1
|
||||
self.cycles_failed += 1
|
||||
self.consecutive_failures += 1
|
||||
if transient:
|
||||
self.errors_transient += 1
|
||||
else:
|
||||
self.errors_other += 1
|
||||
self.last_error = message
|
||||
self._write_heartbeat()
|
||||
|
||||
def touch(self):
|
||||
with self._lock:
|
||||
self._write_heartbeat()
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
now = self._clock()
|
||||
return {
|
||||
"ts": _utc_iso(now),
|
||||
"uptime_s": int(now - self.started),
|
||||
"cycles": self.cycles,
|
||||
"cycles_failed": self.cycles_failed,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"images_ok": self.images_ok,
|
||||
"images_failed": self.images_failed,
|
||||
"errors_transient": self.errors_transient,
|
||||
"errors_other": self.errors_other,
|
||||
"last_success": _utc_iso(self.last_success),
|
||||
"last_success_age_s": None if self.last_success is None else int(now - self.last_success),
|
||||
"last_error": self.last_error,
|
||||
}
|
||||
|
||||
def summary(self) -> str:
|
||||
s = self.snapshot()
|
||||
age = "-" if s["last_success_age_s"] is None else f"{s['last_success_age_s']}s"
|
||||
return (
|
||||
f"uptime={s['uptime_s']}s cycles={s['cycles']} cycles_failed={s['cycles_failed']} "
|
||||
f"consecutive_failures={s['consecutive_failures']} images_ok={s['images_ok']} "
|
||||
f"images_failed={s['images_failed']} errors_transient={s['errors_transient']} "
|
||||
f"errors_other={s['errors_other']} last_success={s['last_success']} last_success_age={age}"
|
||||
)
|
||||
|
||||
def _write_heartbeat(self):
|
||||
if not self.heartbeat_file:
|
||||
return
|
||||
try:
|
||||
tmp = f"{self.heartbeat_file}.tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(self.snapshot(), f)
|
||||
os.replace(tmp, self.heartbeat_file)
|
||||
except OSError as e:
|
||||
print(f"[WARN] heartbeat file not writable: {e}")
|
||||
|
||||
|
||||
def poll_forever(
|
||||
cfg: AppConfig,
|
||||
*,
|
||||
cycle=run_once,
|
||||
init_schema=init_db_schema,
|
||||
sleep=time.sleep,
|
||||
clock=time.time,
|
||||
stats: Optional[PollerStats] = None,
|
||||
max_iterations: Optional[int] = None,
|
||||
) -> PollerStats:
|
||||
"""Hauptschleife. Faengt Fehler je Zyklus ab und wartet mit Backoff - S3- oder DB-Ausfaelle
|
||||
beenden den Prozess nicht mehr. db_init laeuft einmal erfolgreich, danach nicht erneut.
|
||||
Transaktion pro Bild und Statusfolge liegen unveraendert in _process_ready_batch/process_one.
|
||||
"""
|
||||
stats = stats or PollerStats(cfg.heartbeat_file, clock=clock)
|
||||
backoff = Backoff(cfg.backoff_initial_seconds, cfg.backoff_max_seconds)
|
||||
schema_ready = False
|
||||
failing_since: Optional[float] = None
|
||||
last_heartbeat_log = clock()
|
||||
iterations = 0
|
||||
|
||||
while max_iterations is None or iterations < max_iterations:
|
||||
iterations += 1
|
||||
stats.touch()
|
||||
ok_before = stats.images_ok
|
||||
try:
|
||||
if not schema_ready:
|
||||
init_schema(cfg)
|
||||
schema_ready = True
|
||||
n = cycle(cfg, stats)
|
||||
except Exception as e:
|
||||
transient = is_transient_error(e)
|
||||
msg = describe_error(e)
|
||||
delay = backoff.next_delay()
|
||||
stats.cycle_failed(transient, msg)
|
||||
if failing_since is None:
|
||||
failing_since = clock()
|
||||
kind = "transient" if transient else "unexpected"
|
||||
print(
|
||||
f"[CYCLE_FAIL] kind={kind} consecutive={backoff.failures} "
|
||||
f"retry_in={delay:.0f}s error={msg}"
|
||||
)
|
||||
if not transient:
|
||||
traceback.print_exc()
|
||||
n = None
|
||||
else:
|
||||
if failing_since is not None:
|
||||
print(
|
||||
f"[RECOVERED] after_failures={backoff.failures} "
|
||||
f"outage_s={int(clock() - failing_since)}"
|
||||
)
|
||||
failing_since = None
|
||||
backoff.reset()
|
||||
stats.cycle_ok()
|
||||
delay = None
|
||||
|
||||
if clock() - last_heartbeat_log >= cfg.heartbeat_seconds:
|
||||
print(f"[HEARTBEAT] {stats.summary()}")
|
||||
last_heartbeat_log = clock()
|
||||
|
||||
if delay is not None:
|
||||
sleep(delay)
|
||||
elif n == 0 or stats.images_ok == ok_before:
|
||||
# Nichts zu tun - oder nur Saetze, die erneut scheitern: nicht im Leerlauf kreiseln.
|
||||
sleep(cfg.poll_seconds)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
if os.environ.get("DEBUG") == "1":
|
||||
import debugpy
|
||||
debugpy.listen(("0.0.0.0", 5678))
|
||||
print("Waiting for debugger...")
|
||||
debugpy.wait_for_client()
|
||||
|
||||
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
|
||||
cfg = load_config(cfg_path)
|
||||
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
while True:
|
||||
n = run_once(cfg)
|
||||
if n == 0:
|
||||
time.sleep(cfg.poll_seconds)
|
||||
print(
|
||||
f"[START] poll={cfg.poll_seconds}s workers={cfg.parallel_workers} "
|
||||
f"s3_timeout=connect:{cfg.s3_connect_timeout_seconds:g}s/read:{cfg.s3_read_timeout_seconds:g}s "
|
||||
f"s3_retries={cfg.s3_retry_mode}/{cfg.s3_max_attempts} pg_connect_timeout={cfg.pg_connect_timeout_seconds}s "
|
||||
f"backoff={cfg.backoff_initial_seconds:g}-{cfg.backoff_max_seconds:g}s heartbeat={cfg.heartbeat_seconds}s"
|
||||
)
|
||||
poll_forever(cfg)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
-- Normalisiertes Fachmodell: Kamerastatus, Bildeigenschaften, Objektklassifikationen.
|
||||
-- Idempotent; wird vom gundi-receiver beim Start unter Advisory-Lock angewendet.
|
||||
-- Setzt das bestehende Schema remote_cam voraus (die App-Rolle hat kein CREATE auf DB-Ebene).
|
||||
|
||||
-- Statusupdates / Kameraeigenschaften: eine Zeile je Statusmeldung (Zeitreihe).
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.camera_status (
|
||||
status_id bigserial PRIMARY KEY,
|
||||
imei text NOT NULL,
|
||||
reported_at timestamptz NOT NULL,
|
||||
received_at timestamptz NOT NULL DEFAULT now(),
|
||||
iccid text,
|
||||
init_key text,
|
||||
camera_name text,
|
||||
camera_model text,
|
||||
firmware_version text,
|
||||
camera_language text,
|
||||
customer_timezone text,
|
||||
mcc text,
|
||||
mnc text,
|
||||
lac text,
|
||||
cell_id text,
|
||||
camera_network text,
|
||||
battery_pct integer,
|
||||
signal_strength integer,
|
||||
temperature_c integer,
|
||||
coordinate_switch boolean,
|
||||
work_period boolean,
|
||||
work_start text,
|
||||
work_end text,
|
||||
image_format text,
|
||||
image_night text,
|
||||
thumbnail_size text,
|
||||
pir_interval text,
|
||||
time_scan text,
|
||||
lat double precision,
|
||||
lon double precision,
|
||||
settings_changed_at timestamptz,
|
||||
source text NOT NULL CHECK (source IN ('gundi', 'entrance')),
|
||||
source_ref text,
|
||||
raw jsonb
|
||||
);
|
||||
|
||||
-- Legacy hat 173 doppelte (imei, servertime)-Paare, daher keine globale Eindeutigkeit.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS camera_status_gundi_uq
|
||||
ON remote_cam.camera_status (imei, reported_at) WHERE source = 'gundi';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS camera_status_entrance_uq
|
||||
ON remote_cam.camera_status (source_ref) WHERE source = 'entrance';
|
||||
CREATE INDEX IF NOT EXISTS camera_status_imei_reported_idx
|
||||
ON remote_cam.camera_status (imei, reported_at DESC);
|
||||
|
||||
-- Bildeigenschaften.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.image (
|
||||
image_uuid uuid PRIMARY KEY,
|
||||
imei text NOT NULL,
|
||||
servertime timestamptz NOT NULL,
|
||||
ocr_ts timestamptz,
|
||||
ocr_state text NOT NULL DEFAULT 'PENDING' CHECK (ocr_state IN ('PENDING', 'OK', 'FAIL')),
|
||||
ocr_attempts integer NOT NULL DEFAULT 0,
|
||||
ocr_error text,
|
||||
camera_ts timestamptz,
|
||||
camera_name text,
|
||||
camera_make text,
|
||||
lat double precision,
|
||||
lon double precision,
|
||||
size_bytes integer,
|
||||
processed_key text,
|
||||
thumbnail_key text,
|
||||
source text NOT NULL CHECK (source IN ('gundi', 'entrance')),
|
||||
source_ref text,
|
||||
imported_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
raw jsonb
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS image_imei_servertime_idx
|
||||
ON remote_cam.image (imei, servertime DESC);
|
||||
CREATE INDEX IF NOT EXISTS image_servertime_idx
|
||||
ON remote_cam.image (servertime DESC);
|
||||
CREATE INDEX IF NOT EXISTS image_ocr_pending_idx
|
||||
ON remote_cam.image (updated_at) WHERE ocr_state = 'PENDING';
|
||||
|
||||
-- Objektklassifikationen.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.detection (
|
||||
detection_id bigserial PRIMARY KEY,
|
||||
image_uuid uuid NOT NULL REFERENCES remote_cam.image (image_uuid) ON DELETE CASCADE,
|
||||
category text NOT NULL,
|
||||
confidence double precision CHECK (confidence BETWEEN 0 AND 1),
|
||||
bbox_x integer,
|
||||
bbox_y integer,
|
||||
bbox_w integer,
|
||||
bbox_h integer,
|
||||
model_version text,
|
||||
source text NOT NULL CHECK (source IN ('gundi', 'entrance'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS detection_image_confidence_idx
|
||||
ON remote_cam.detection (image_uuid, confidence DESC);
|
||||
CREATE INDEX IF NOT EXISTS detection_category_idx
|
||||
ON remote_cam.detection (category);
|
||||
|
||||
-- Einheitliches Label je Rohkategorie. detection.category bleibt die Rohbezeichnung der
|
||||
-- Quelle (ICU-Gundi: "Deer-Like", Legacy-Klassifikator: "deer"); Auswertungen nutzen das Label.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.category_label (
|
||||
category text PRIMARY KEY,
|
||||
label text NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO remote_cam.category_label (category, label) VALUES
|
||||
('Deer-Like', 'deer'), ('deer', 'deer'),
|
||||
('person', 'human'), ('human', 'human'),
|
||||
('car', 'machine'), ('machine', 'machine'),
|
||||
('Bird', 'bird'), ('bird', 'bird'),
|
||||
('Fox', 'fox'), ('fox', 'fox'),
|
||||
('Wild Boar', 'wild boar'), ('wild boar', 'wild boar'),
|
||||
('racoon', 'raccoon'), ('raccoon', 'raccoon')
|
||||
ON CONFLICT (category) DO NOTHING;
|
||||
|
||||
-- Ohne Eintrag in category_label gilt die kleingeschriebene Rohbezeichnung.
|
||||
CREATE OR REPLACE VIEW remote_cam.detection_labeled AS
|
||||
SELECT d.*, COALESCE(cl.label, lower(d.category)) AS label
|
||||
FROM remote_cam.detection d
|
||||
LEFT JOIN remote_cam.category_label cl ON cl.category = d.category;
|
||||
|
||||
-- Gundi-Bilder, die waehrend des Parallelbetriebs auch ueber den Legacy-Pfad kamen und
|
||||
-- zugunsten des Legacy-Zwillings (traegt die KI-Ergebnisse) entfernt wurden. Haelt die
|
||||
-- geloeschte image-Zeile samt Detections fuer Nachvollziehbarkeit und Rollback.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.image_superseded (
|
||||
image_uuid uuid PRIMARY KEY,
|
||||
superseded_by uuid NOT NULL,
|
||||
removed_at timestamptz NOT NULL DEFAULT now(),
|
||||
image_row jsonb NOT NULL,
|
||||
detections jsonb
|
||||
);
|
||||
|
||||
-- Transportschicht des Push-Pfads: Rohevents + Zustandsmaschine.
|
||||
CREATE TABLE IF NOT EXISTS remote_cam.gundi_inbox (
|
||||
object_id uuid PRIMARY KEY,
|
||||
event_type text NOT NULL,
|
||||
source text,
|
||||
recorded_at timestamptz,
|
||||
payload jsonb NOT NULL,
|
||||
received_at timestamptz NOT NULL DEFAULT now(),
|
||||
attachment_key text,
|
||||
attachment_received_at timestamptz,
|
||||
state text NOT NULL CHECK (state IN (
|
||||
'AWAITING_ATTACHMENT', 'RECEIVED', 'INGESTED',
|
||||
'FAILED', 'ABANDONED', 'IGNORED')),
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
last_error text,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS gundi_inbox_open_idx
|
||||
ON remote_cam.gundi_inbox (state, updated_at)
|
||||
WHERE state IN ('AWAITING_ATTACHMENT', 'RECEIVED');
|
||||
|
||||
-- Bild + der zum Aufnahme-Serverzeitpunkt gueltige Kamerastatus.
|
||||
-- Tie-Breaker bei gleichem reported_at: der aus derselben Legacy-Zeile abgespaltene Status.
|
||||
CREATE OR REPLACE VIEW remote_cam.image_with_status AS
|
||||
SELECT
|
||||
i.image_uuid,
|
||||
i.imei,
|
||||
i.servertime,
|
||||
i.ocr_ts,
|
||||
i.ocr_state,
|
||||
i.camera_ts,
|
||||
i.camera_name,
|
||||
i.camera_make,
|
||||
i.lat,
|
||||
i.lon,
|
||||
i.size_bytes,
|
||||
i.processed_key,
|
||||
i.thumbnail_key,
|
||||
i.source,
|
||||
i.imported_at,
|
||||
s.status_id,
|
||||
s.reported_at AS status_reported_at,
|
||||
s.iccid,
|
||||
s.init_key,
|
||||
s.camera_name AS status_camera_name,
|
||||
s.camera_model,
|
||||
s.firmware_version,
|
||||
s.camera_language,
|
||||
s.customer_timezone,
|
||||
s.mcc,
|
||||
s.mnc,
|
||||
s.lac,
|
||||
s.cell_id,
|
||||
s.camera_network,
|
||||
s.battery_pct,
|
||||
s.signal_strength,
|
||||
s.temperature_c,
|
||||
s.coordinate_switch,
|
||||
s.work_period,
|
||||
s.work_start,
|
||||
s.work_end,
|
||||
s.image_format,
|
||||
s.image_night,
|
||||
s.thumbnail_size,
|
||||
s.pir_interval,
|
||||
s.time_scan,
|
||||
s.settings_changed_at
|
||||
FROM remote_cam.image i
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT cs.*
|
||||
FROM remote_cam.camera_status cs
|
||||
WHERE cs.imei = i.imei
|
||||
AND cs.reported_at <= i.servertime
|
||||
ORDER BY cs.reported_at DESC,
|
||||
(cs.source_ref = i.image_uuid::text) DESC NULLS LAST,
|
||||
cs.status_id DESC
|
||||
LIMIT 1
|
||||
) s ON TRUE;
|
||||
@@ -0,0 +1,106 @@
|
||||
-- Split der Legacy-Daten ins Fachmodell: jede remote_cam.metadata-Zeile ergibt
|
||||
-- ein image und ein camera_status; remote_cam.category (Score >= 0.5) wird zu detection.
|
||||
--
|
||||
-- Erwartet die Temp-Tabelle _mig_batch(metadata_uuid text), die tools/migrate_legacy.py
|
||||
-- je Batch fuellt. Wiederholbar: bereits migrierte Zeilen werden uebersprungen.
|
||||
-- Alttabellen werden nur gelesen.
|
||||
|
||||
-- step: image
|
||||
INSERT INTO remote_cam.image (
|
||||
image_uuid, imei, servertime, ocr_ts, ocr_state, ocr_error, camera_ts,
|
||||
camera_name, camera_make, lat, lon, size_bytes, processed_key, thumbnail_key,
|
||||
source, source_ref, imported_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
m.metadata_uuid::uuid,
|
||||
m.imei,
|
||||
m.servertime,
|
||||
m.ocr_ts,
|
||||
CASE WHEN m.ocr_ts IS NULL THEN 'FAIL' ELSE 'OK' END,
|
||||
CASE WHEN m.ocr_ts IS NULL THEN 'legacy: no ocr_ts' END,
|
||||
m."ImageCreationDate",
|
||||
m."CameraName",
|
||||
m."CameraModel",
|
||||
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Latitude" END,
|
||||
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Longitude" END,
|
||||
m."SizeInByte",
|
||||
j.processed_jpg_key,
|
||||
j.thumbnail_key,
|
||||
'entrance',
|
||||
m.metadata_uuid,
|
||||
COALESCE(m.import_ts, now()),
|
||||
now()
|
||||
FROM _mig_batch b
|
||||
JOIN remote_cam.metadata m ON m.metadata_uuid = b.metadata_uuid
|
||||
LEFT JOIN remote_cam.import_job j ON j.image_uuid = m.metadata_uuid
|
||||
WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
|
||||
ON CONFLICT (image_uuid) DO NOTHING;
|
||||
|
||||
-- step: camera_status
|
||||
INSERT INTO remote_cam.camera_status (
|
||||
imei, reported_at, received_at, iccid, init_key, camera_name, camera_model,
|
||||
firmware_version, camera_language, customer_timezone, mcc, mnc, lac, camera_network,
|
||||
battery_pct, signal_strength, temperature_c, coordinate_switch, work_period,
|
||||
work_start, work_end, image_format, image_night, thumbnail_size, pir_interval, time_scan,
|
||||
lat, lon, settings_changed_at, source, source_ref
|
||||
)
|
||||
SELECT
|
||||
m.imei,
|
||||
m.servertime,
|
||||
COALESCE(m.import_ts, now()),
|
||||
m.iccid,
|
||||
m."INITKey",
|
||||
m."CameraName",
|
||||
m."CameraModel",
|
||||
m."FirmwareVersion",
|
||||
m."CameraLanguage",
|
||||
m."CustomerTimezone",
|
||||
m."MobileCountryCode",
|
||||
m."MobileNetworkCode",
|
||||
m."LocationAreaCode",
|
||||
m."CameraNetwork",
|
||||
m."BatteryStatus",
|
||||
m."SignalStrength",
|
||||
m."Temperature",
|
||||
m."CoordinateSwitch",
|
||||
m."WorkPeriod",
|
||||
m."WorkStart",
|
||||
m."WorkEnd",
|
||||
m."ImageFormat",
|
||||
m."ImageNight",
|
||||
m."ThumbnailSize",
|
||||
m."PIRInterval",
|
||||
m."TimeScan",
|
||||
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Latitude" END,
|
||||
CASE WHEN m."Latitude" = 0 AND m."Longitude" = 0 THEN NULL ELSE m."Longitude" END,
|
||||
m."DateTimeLastSettings",
|
||||
'entrance',
|
||||
m.metadata_uuid
|
||||
FROM _mig_batch b
|
||||
JOIN remote_cam.metadata m ON m.metadata_uuid = b.metadata_uuid
|
||||
WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL
|
||||
ON CONFLICT (source_ref) WHERE source = 'entrance' DO NOTHING;
|
||||
|
||||
-- step: detection
|
||||
-- remote_cam.category ist ein Score-Vektor ueber alle Klassen, detection haelt nur Erkanntes:
|
||||
-- uebernommen werden Klassen ab Score 0.5, ohne '' und 'none' (leeres Bild = keine Detection).
|
||||
-- Ausnahme Datenschutz: war 'human' oder 'machine' die Top-Kategorie des Bildes, wird sie auch
|
||||
-- unter 0.5 uebernommen -- das Portal sperrt solche Bilder fuer Nicht-Admins (wie bisher ueber
|
||||
-- die Top-Kategorie). Tiebreaker wie die fruehere image_top_category: Score, dann Name.
|
||||
INSERT INTO remote_cam.detection (image_uuid, category, confidence, model_version, source)
|
||||
SELECT i.image_uuid, c.category, c.score, c.version, 'entrance'
|
||||
FROM _mig_batch b
|
||||
JOIN remote_cam.category c ON c.image_uuid = b.metadata_uuid
|
||||
JOIN remote_cam.image i ON i.image_uuid = b.metadata_uuid::uuid
|
||||
WHERE c.category IS NOT NULL
|
||||
AND c.category NOT IN ('', 'none')
|
||||
AND (c.score >= 0.5
|
||||
OR (c.category IN ('human', 'machine')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM remote_cam.category c2
|
||||
WHERE c2.image_uuid = c.image_uuid
|
||||
AND (c2.score > c.score OR (c2.score = c.score AND c2.category < c.category)))))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM remote_cam.detection d
|
||||
WHERE d.image_uuid = i.image_uuid AND d.source = 'entrance'
|
||||
);
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Parallelbetrieb Legacy + Gundi: dasselbe Bild liegt doppelt in remote_cam.image.
|
||||
-- Behalten wird der Legacy-Zwilling (source='entrance'): an seiner UUID haengen die
|
||||
-- KI-Ergebnisse (ai_image/ai_object). Der Gundi-Zwilling wandert nach image_superseded.
|
||||
--
|
||||
-- Zwilling = gleiche imei, gleiche Dateigroesse, servertime hoechstens 10 min auseinander.
|
||||
-- Gemessen am 2026-10-16: 4291 Paare, jedes Bild hat hoechstens einen Partner.
|
||||
-- Nur Paare, deren Legacy-Seite schon migriert ist; Paare mit mehreren Kandidaten bleiben
|
||||
-- unangetastet. Wiederholbar. tools/migrate_legacy.py --dedupe fuehrt die Schritte in EINER
|
||||
-- Transaktion aus.
|
||||
|
||||
-- step: twins
|
||||
CREATE TEMP TABLE _twins ON COMMIT DROP AS
|
||||
WITH pairs AS (
|
||||
SELECT g.image_uuid AS gundi_uuid, e.image_uuid AS legacy_uuid
|
||||
FROM remote_cam.image g
|
||||
JOIN remote_cam.image e
|
||||
ON e.source = 'entrance'
|
||||
AND e.imei = g.imei
|
||||
AND e.size_bytes = g.size_bytes
|
||||
AND e.servertime BETWEEN g.servertime - interval '10 minutes'
|
||||
AND g.servertime + interval '10 minutes'
|
||||
WHERE g.source = 'gundi'
|
||||
)
|
||||
SELECT gundi_uuid, legacy_uuid
|
||||
FROM (SELECT p.*,
|
||||
count(*) OVER (PARTITION BY p.gundi_uuid) AS n_gundi,
|
||||
count(*) OVER (PARTITION BY p.legacy_uuid) AS n_legacy
|
||||
FROM pairs p) x
|
||||
WHERE n_gundi = 1 AND n_legacy = 1;
|
||||
|
||||
-- step: archive
|
||||
INSERT INTO remote_cam.image_superseded (image_uuid, superseded_by, image_row, detections)
|
||||
SELECT t.gundi_uuid, t.legacy_uuid, to_jsonb(i),
|
||||
(SELECT jsonb_agg(to_jsonb(d)) FROM remote_cam.detection d WHERE d.image_uuid = t.gundi_uuid)
|
||||
FROM _twins t
|
||||
JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid
|
||||
ON CONFLICT (image_uuid) DO NOTHING;
|
||||
|
||||
-- step: delete
|
||||
-- detection haengt per ON DELETE CASCADE am image.
|
||||
DELETE FROM remote_cam.image i
|
||||
USING _twins t
|
||||
WHERE i.image_uuid = t.gundi_uuid;
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
|
||||
|
||||
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
||||
|
||||
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
||||
"""
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import psycopg # noqa: E402
|
||||
from botocore.exceptions import ClientError # noqa: E402
|
||||
|
||||
import gundi_pipeline as gp # noqa: E402
|
||||
import gundi_receiver as gr # noqa: E402
|
||||
|
||||
|
||||
def s3_unavailable() -> ClientError:
|
||||
return ClientError(
|
||||
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
|
||||
"PutObject",
|
||||
)
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, rows=None, rowcount=1):
|
||||
self.executed: list[tuple[str, tuple]] = []
|
||||
self._rows = list(rows or [])
|
||||
self.rowcount = rowcount
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
self.executed.append((" ".join(sql.split()), params))
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows.pop(0) if self._rows else None
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, cursor: FakeCursor):
|
||||
self._cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self._cursor
|
||||
|
||||
|
||||
def fake_connect(cursor: FakeCursor):
|
||||
return lambda dsn: FakeConn(cursor)
|
||||
|
||||
|
||||
GCFG = SimpleNamespace(inbox_prefix="icu/gundi/inbox/", max_attempts=3)
|
||||
|
||||
|
||||
class AttachImageTests(unittest.TestCase):
|
||||
def test_s3_down_raises_without_db_lock_and_without_update(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||
s3 = mock.Mock()
|
||||
s3.put_bytes.side_effect = s3_unavailable()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
with self.assertRaises(ClientError):
|
||||
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
|
||||
sqls = [sql for sql, _ in cur.executed]
|
||||
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
|
||||
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
|
||||
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
|
||||
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
|
||||
|
||||
def test_success_stores_then_updates(self):
|
||||
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
||||
s3.put_bytes.assert_called_once_with("icu/gundi/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
||||
|
||||
def test_already_received_is_duplicate_without_s3(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
|
||||
s3 = mock.Mock()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
|
||||
s3.put_bytes.assert_not_called()
|
||||
|
||||
def test_parallel_upload_lost_the_race_is_duplicate(self):
|
||||
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
|
||||
|
||||
def test_unknown_or_wrong_type_is_lookup_error(self):
|
||||
for rows in ([], [("camera_status_report", "INGESTED")]):
|
||||
cur = FakeCursor(rows=rows)
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||
with self.assertRaises(LookupError):
|
||||
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
|
||||
|
||||
|
||||
def bare_pipeline() -> gp.Pipeline:
|
||||
p = gp.Pipeline.__new__(gp.Pipeline)
|
||||
p.cfg = SimpleNamespace(pg_dsn="dsn")
|
||||
p.gcfg = GCFG
|
||||
p._inflight = set()
|
||||
p._inflight_lock = threading.Lock()
|
||||
return p
|
||||
|
||||
|
||||
class FailureRecordingTests(unittest.TestCase):
|
||||
def run_failure(self, method: str, err: Exception) -> str:
|
||||
cur = FakeCursor(rows=[(1, "RECEIVED")])
|
||||
p = bare_pipeline()
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
|
||||
getattr(p, method)("id-1", err)
|
||||
self.assertEqual(len(cur.executed), 1)
|
||||
return cur.executed[0][0]
|
||||
|
||||
def test_ingest_transient_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
|
||||
self.assertNotIn("attempts + 1", sql)
|
||||
self.assertNotIn("FAILED", sql)
|
||||
|
||||
def test_ingest_db_outage_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
|
||||
self.assertNotIn("attempts + 1", sql)
|
||||
|
||||
def test_ingest_permanent_counts_attempt(self):
|
||||
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
|
||||
self.assertIn("attempts + 1", sql)
|
||||
self.assertIn("'FAILED'", sql)
|
||||
|
||||
def test_ocr_transient_does_not_count_attempt(self):
|
||||
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
|
||||
self.assertNotIn("ocr_attempts + 1", sql)
|
||||
self.assertNotIn("'FAIL'", sql)
|
||||
|
||||
def test_ocr_permanent_counts_attempt(self):
|
||||
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
|
||||
self.assertIn("ocr_attempts + 1", sql)
|
||||
self.assertIn("'FAIL'", sql)
|
||||
|
||||
|
||||
class BacklogAlarmTests(unittest.TestCase):
|
||||
def run_check(self, inbox_row, image_row, pipeline=None):
|
||||
cur = FakeCursor(rows=[inbox_row, image_row])
|
||||
p = pipeline or bare_pipeline()
|
||||
p.gcfg = SimpleNamespace(alarm_after_seconds=1800)
|
||||
p._last_alarm = 0.0
|
||||
out = []
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))):
|
||||
p.check_backlog()
|
||||
return p, cur, out
|
||||
|
||||
def test_silent_when_nothing_is_wrong(self):
|
||||
_, _, out = self.run_check((0, 0, 0), (0, 0))
|
||||
self.assertEqual(out, [])
|
||||
|
||||
def test_alarm_for_stuck_and_new_failures(self):
|
||||
_, _, out = self.run_check((2, 0, 1), (0, 3))
|
||||
self.assertEqual(len(out), 1)
|
||||
self.assertIn("[GUNDI][ALARM]", out[0])
|
||||
self.assertIn("received_haengt=2", out[0])
|
||||
self.assertIn("failed_24h=1", out[0])
|
||||
self.assertIn("ocr_fail_24h=3", out[0])
|
||||
|
||||
def test_failures_older_than_24h_are_not_counted(self):
|
||||
_, cur, _ = self.run_check((0, 0, 0), (0, 0))
|
||||
sqls = " ".join(sql for sql, _ in cur.executed)
|
||||
self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls)
|
||||
self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls)
|
||||
|
||||
def test_alarm_is_throttled(self):
|
||||
p, _, first = self.run_check((1, 0, 0), (0, 0))
|
||||
self.assertEqual(len(first), 1)
|
||||
cur = FakeCursor(rows=[(1, 0, 0), (0, 0)])
|
||||
out = []
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)):
|
||||
p.check_backlog() # direkt danach -> unterdrueckt
|
||||
self.assertEqual(out, [])
|
||||
|
||||
|
||||
class SubmitDedupeTests(unittest.TestCase):
|
||||
def test_same_id_is_queued_once_until_done(self):
|
||||
p = bare_pipeline()
|
||||
pool = mock.Mock()
|
||||
calls = []
|
||||
p.ingest = lambda arg: calls.append(arg)
|
||||
p.ingest.__name__ = "ingest"
|
||||
|
||||
p._submit(pool, p.ingest, "a")
|
||||
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
|
||||
p._submit(pool, p.ingest, "b")
|
||||
self.assertEqual(pool.submit.call_count, 2)
|
||||
|
||||
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
|
||||
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
|
||||
p._guarded(token, fn, arg)
|
||||
self.assertEqual(calls, ["a"])
|
||||
p._submit(pool, p.ingest, "a")
|
||||
self.assertEqual(pool.submit.call_count, 3)
|
||||
|
||||
def test_failing_worker_releases_the_id(self):
|
||||
p = bare_pipeline()
|
||||
pool = mock.Mock()
|
||||
|
||||
def boom(arg):
|
||||
raise RuntimeError("x")
|
||||
|
||||
p._submit(pool, boom, "a")
|
||||
token, fn, arg = pool.submit.call_args.args[1:4]
|
||||
with mock.patch("builtins.print"):
|
||||
p._guarded(token, fn, arg)
|
||||
self.assertEqual(p._inflight, set())
|
||||
|
||||
|
||||
class ReceiverStatusCodeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
|
||||
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
|
||||
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
|
||||
cls.port = cls.server.server_address[1]
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
||||
cls.thread.start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.shutdown()
|
||||
cls.server.server_close()
|
||||
|
||||
def post(self, path, body, content_type):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
try:
|
||||
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
|
||||
resp = conn.getresponse()
|
||||
return resp.status, resp.getheader("Retry-After"), resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def attachment(self):
|
||||
boundary = "xx"
|
||||
body = (
|
||||
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
|
||||
"Content-Type: image/jpeg\r\n\r\n"
|
||||
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
|
||||
path = f"/v2/events/{uuid.uuid4()}/attachments/"
|
||||
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
|
||||
|
||||
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
|
||||
status, retry_after, _ = self.attachment()
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||
|
||||
def test_unexpected_error_on_upload_stays_500(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
|
||||
status, retry_after, _ = self.attachment()
|
||||
self.assertEqual(status, 500)
|
||||
self.assertIsNone(retry_after)
|
||||
|
||||
def test_db_outage_on_event_gives_503(self):
|
||||
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
|
||||
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
|
||||
mock.patch("builtins.print"):
|
||||
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||
|
||||
def test_unknown_object_id_is_still_404(self):
|
||||
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
|
||||
status, _, _ = self.attachment()
|
||||
self.assertEqual(status, 404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,95 @@
|
||||
"""tools/migrate_legacy: Schrittdateien und --dedupe-Ablauf, ohne DB.
|
||||
|
||||
python -m unittest discover -s tests -v
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from tools import migrate_legacy as ml # noqa: E402
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, n_twins, deleted=None):
|
||||
self.executed = []
|
||||
self.n_twins = n_twins
|
||||
self.deleted = n_twins if deleted is None else deleted
|
||||
self.rowcount = 0
|
||||
self._next = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.executed.append(sql)
|
||||
if "count(*) FROM _twins" in sql:
|
||||
self._next = (self.n_twins,)
|
||||
elif "min(i.servertime)" in sql:
|
||||
self._next = (None, None)
|
||||
elif sql.lstrip().startswith("INSERT INTO remote_cam.image_superseded"):
|
||||
self.rowcount = self.n_twins
|
||||
elif "DELETE FROM remote_cam.image" in sql:
|
||||
self.rowcount = self.deleted
|
||||
|
||||
def fetchone(self):
|
||||
return self._next
|
||||
|
||||
|
||||
def fake_conn(cur):
|
||||
conn = mock.Mock()
|
||||
conn.cursor.return_value = cur
|
||||
return conn
|
||||
|
||||
|
||||
class StepFiles(unittest.TestCase):
|
||||
def test_migration_steps(self):
|
||||
steps = dict(ml.load_steps())
|
||||
self.assertEqual(list(steps), ["image", "camera_status", "detection"])
|
||||
det = steps["detection"]
|
||||
self.assertIn("c.score >= 0.5", det)
|
||||
self.assertIn("NOT IN ('', 'none')", det)
|
||||
self.assertIn("c.category IN ('human', 'machine')", det)
|
||||
self.assertIn("c.category IN ('human', 'machine')", ml.DETECTION_FILTER)
|
||||
|
||||
def test_dedupe_steps(self):
|
||||
steps = dict(ml.load_steps("003_dedupe_gundi_twins.sql"))
|
||||
self.assertEqual(list(steps), ["twins", "archive", "delete"])
|
||||
self.assertIn("e.size_bytes = g.size_bytes", steps["twins"])
|
||||
self.assertIn("n_gundi = 1 AND n_legacy = 1", steps["twins"])
|
||||
|
||||
|
||||
class Dedupe(unittest.TestCase):
|
||||
def test_dry_run_changes_nothing(self):
|
||||
cur = FakeCursor(n_twins=5)
|
||||
conn = fake_conn(cur)
|
||||
ml.dedupe(conn, dry_run=True)
|
||||
self.assertFalse(any("image_superseded" in s or "DELETE" in s for s in cur.executed))
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
def test_archives_then_deletes(self):
|
||||
cur = FakeCursor(n_twins=5)
|
||||
conn = fake_conn(cur)
|
||||
ml.dedupe(conn, dry_run=False)
|
||||
archive = next(i for i, s in enumerate(cur.executed) if "INSERT INTO remote_cam.image_superseded" in s)
|
||||
delete = next(i for i, s in enumerate(cur.executed) if "DELETE FROM remote_cam.image" in s)
|
||||
self.assertLess(archive, delete)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
def test_count_mismatch_rolls_back(self):
|
||||
cur = FakeCursor(n_twins=5, deleted=4)
|
||||
conn = fake_conn(cur)
|
||||
with self.assertRaises(RuntimeError):
|
||||
ml.dedupe(conn, dry_run=False)
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,440 @@
|
||||
"""Backoff und Fehlerbehandlung des Legacy-Pollers mit simulierten S3-/DB-Stoerungen.
|
||||
|
||||
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
||||
|
||||
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import psycopg # noqa: E402
|
||||
from botocore.exceptions import ( # noqa: E402
|
||||
ClientError,
|
||||
ConnectTimeoutError,
|
||||
ConnectionClosedError,
|
||||
EndpointConnectionError,
|
||||
ReadTimeoutError,
|
||||
)
|
||||
|
||||
import main # noqa: E402
|
||||
|
||||
|
||||
def client_error(code: str, http: int, op: str = "ListObjectsV2") -> ClientError:
|
||||
return ClientError({"Error": {"Code": code, "Message": code}, "ResponseMetadata": {"HTTPStatusCode": http}}, op)
|
||||
|
||||
|
||||
def s3_unavailable() -> ClientError:
|
||||
return client_error("ServiceUnavailable", 503)
|
||||
|
||||
|
||||
def make_cfg(tmpdir: str, **overrides) -> main.AppConfig:
|
||||
params = dict(
|
||||
s3_endpoint="https://s3.invalid/",
|
||||
s3_access_key="test",
|
||||
s3_secret_key="test",
|
||||
s3_bucket="bucket",
|
||||
pg_dsn="postgresql://invalid/db",
|
||||
poll_seconds=30,
|
||||
parallel_workers=1,
|
||||
backoff_initial_seconds=5,
|
||||
backoff_max_seconds=300,
|
||||
heartbeat_seconds=300,
|
||||
heartbeat_file=os.path.join(tmpdir, "heartbeat.json"),
|
||||
)
|
||||
params.update(overrides)
|
||||
return main.AppConfig(**params)
|
||||
|
||||
|
||||
class FakeClock:
|
||||
"""Zeit laeuft nur durch sleep() - Tests laufen ohne echtes Warten."""
|
||||
|
||||
def __init__(self, start: float = 1_000_000.0):
|
||||
self.now = start
|
||||
self.sleeps: list[float] = []
|
||||
|
||||
def time(self) -> float:
|
||||
return self.now
|
||||
|
||||
def sleep(self, seconds: float):
|
||||
self.sleeps.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
class ScriptedCycle:
|
||||
"""Zyklus-Ersatz: arbeitet eine Liste ab (Exception -> werfen, int -> zurueckgeben)."""
|
||||
|
||||
def __init__(self, steps):
|
||||
self.steps = list(steps)
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, cfg, stats):
|
||||
self.calls += 1
|
||||
step = self.steps.pop(0) if self.steps else 0
|
||||
if isinstance(step, BaseException):
|
||||
raise step
|
||||
if callable(step):
|
||||
return step(stats)
|
||||
return step
|
||||
|
||||
|
||||
class FakeS3:
|
||||
"""Simulierter S3-Client: Listing liefert Objekte oder wirft den angegebenen Fehler."""
|
||||
|
||||
def __init__(self, objects=None, list_error: BaseException = None):
|
||||
self.objects = objects or []
|
||||
self.list_error = list_error
|
||||
|
||||
def list_entrance_objects(self):
|
||||
if self.list_error is not None:
|
||||
raise self.list_error
|
||||
yield from self.objects
|
||||
|
||||
|
||||
class FakeConn:
|
||||
"""Simulierte psycopg-Verbindung; die SQL-Funktionen werden in den Tests gepatcht."""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
@contextlib.contextmanager
|
||||
def transaction(self):
|
||||
yield
|
||||
|
||||
def cursor(self):
|
||||
return contextlib.nullcontext(object())
|
||||
|
||||
|
||||
def entrance_objects(*uuids):
|
||||
old = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
objs = []
|
||||
for u in uuids:
|
||||
objs.append({"Key": f"icu/entrance/{u}.jpg", "LastModified": old})
|
||||
objs.append({"Key": f"icu/entrance/{u}.json", "LastModified": old})
|
||||
return objs
|
||||
|
||||
|
||||
UUIDS = [f"00000000-0000-4000-8000-00000000000{i}" for i in range(1, 4)]
|
||||
|
||||
|
||||
class BackoffTest(unittest.TestCase):
|
||||
def test_doubles_up_to_maximum(self):
|
||||
b = main.Backoff(5, 300)
|
||||
self.assertEqual([b.next_delay() for _ in range(9)], [5, 10, 20, 40, 80, 160, 300, 300, 300])
|
||||
|
||||
def test_reset_starts_again_at_initial(self):
|
||||
b = main.Backoff(5, 300)
|
||||
for _ in range(4):
|
||||
b.next_delay()
|
||||
b.reset()
|
||||
self.assertEqual(b.failures, 0)
|
||||
self.assertEqual(b.next_delay(), 5)
|
||||
|
||||
def test_long_outage_does_not_overflow(self):
|
||||
b = main.Backoff(5, 300)
|
||||
for _ in range(5000):
|
||||
delay = b.next_delay()
|
||||
self.assertEqual(delay, 300)
|
||||
|
||||
|
||||
class ErrorClassificationTest(unittest.TestCase):
|
||||
def test_transient(self):
|
||||
cases = [
|
||||
s3_unavailable(),
|
||||
client_error("SlowDown", 503),
|
||||
client_error("InternalError", 500),
|
||||
client_error("TooManyRequests", 429),
|
||||
client_error("Whatever", 502),
|
||||
EndpointConnectionError(endpoint_url="https://s3.invalid"),
|
||||
ConnectTimeoutError(endpoint_url="https://s3.invalid"),
|
||||
ReadTimeoutError(endpoint_url="https://s3.invalid"),
|
||||
ConnectionClosedError(endpoint_url="https://s3.invalid"),
|
||||
psycopg.OperationalError("connection refused"),
|
||||
psycopg.errors.AdminShutdown("terminating connection due to administrator command"),
|
||||
ConnectionResetError(),
|
||||
TimeoutError(),
|
||||
]
|
||||
for exc in cases:
|
||||
with self.subTest(exc=repr(exc)):
|
||||
self.assertTrue(main.is_transient_error(exc))
|
||||
|
||||
def test_unexpected(self):
|
||||
cases = [
|
||||
client_error("AccessDenied", 403),
|
||||
client_error("NoSuchKey", 404, "GetObject"),
|
||||
client_error("NoSuchBucket", 404),
|
||||
psycopg.ProgrammingError("syntax error"),
|
||||
psycopg.errors.InsufficientPrivilege("permission denied"),
|
||||
KeyError("status"),
|
||||
ValueError("metadata.servertime missing"),
|
||||
]
|
||||
for exc in cases:
|
||||
with self.subTest(exc=repr(exc)):
|
||||
self.assertFalse(main.is_transient_error(exc))
|
||||
|
||||
def test_follows_explicit_cause_only(self):
|
||||
try:
|
||||
try:
|
||||
raise s3_unavailable()
|
||||
except ClientError as e:
|
||||
raise RuntimeError("wrapped") from e
|
||||
except RuntimeError as wrapped:
|
||||
self.assertTrue(main.is_transient_error(wrapped))
|
||||
|
||||
# Ein Bug im Fehlerpfad (nur __context__) bleibt ein unerwarteter Fehler.
|
||||
try:
|
||||
try:
|
||||
raise s3_unavailable()
|
||||
except ClientError:
|
||||
raise KeyError("bug in handler")
|
||||
except KeyError as bug:
|
||||
self.assertFalse(main.is_transient_error(bug))
|
||||
|
||||
|
||||
class PollForeverTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
self.clock = FakeClock()
|
||||
self.out = io.StringIO()
|
||||
self.err = io.StringIO()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def run_loop(self, cycle, iterations, init_schema=lambda cfg: None):
|
||||
with contextlib.redirect_stdout(self.out), contextlib.redirect_stderr(self.err):
|
||||
return main.poll_forever(
|
||||
self.cfg,
|
||||
cycle=cycle,
|
||||
init_schema=init_schema,
|
||||
sleep=self.clock.sleep,
|
||||
clock=self.clock.time,
|
||||
max_iterations=iterations,
|
||||
)
|
||||
|
||||
def test_s3_outage_backs_off_and_recovers(self):
|
||||
cycle = ScriptedCycle([s3_unavailable(), s3_unavailable(), s3_unavailable(), 0, 0, s3_unavailable(), 0])
|
||||
stats = self.run_loop(cycle, 7)
|
||||
self.assertEqual(self.clock.sleeps, [5, 10, 20, 30, 30, 5, 30])
|
||||
self.assertEqual(stats.cycles, 7)
|
||||
self.assertEqual(stats.cycles_failed, 4)
|
||||
self.assertEqual(stats.errors_transient, 4)
|
||||
self.assertEqual(stats.errors_other, 0)
|
||||
self.assertEqual(stats.consecutive_failures, 0)
|
||||
self.assertIsNotNone(stats.last_success)
|
||||
log = self.out.getvalue()
|
||||
self.assertIn("[CYCLE_FAIL] kind=transient consecutive=1 retry_in=5s error=ClientError", log)
|
||||
self.assertIn("[RECOVERED] after_failures=3 outage_s=35", log)
|
||||
self.assertNotIn("Traceback", self.err.getvalue())
|
||||
|
||||
def test_backoff_is_capped_during_long_outage(self):
|
||||
stats = self.run_loop(ScriptedCycle([s3_unavailable()] * 12), 12)
|
||||
self.assertEqual(self.clock.sleeps[-4:], [300, 300, 300, 300])
|
||||
self.assertEqual(stats.consecutive_failures, 12)
|
||||
|
||||
def test_db_unavailable_at_start_is_retried_then_schema_init_runs_once(self):
|
||||
calls = []
|
||||
|
||||
def init_schema(cfg):
|
||||
calls.append(1)
|
||||
if len(calls) <= 2:
|
||||
raise psycopg.OperationalError("could not connect to server")
|
||||
|
||||
cycle = ScriptedCycle([0, 0, 0])
|
||||
stats = self.run_loop(cycle, 5, init_schema=init_schema)
|
||||
self.assertEqual(len(calls), 3)
|
||||
self.assertEqual(cycle.calls, 3)
|
||||
self.assertEqual(self.clock.sleeps, [5, 10, 30, 30, 30])
|
||||
self.assertEqual(stats.errors_transient, 2)
|
||||
|
||||
def test_unexpected_error_is_logged_with_traceback_but_does_not_crash(self):
|
||||
cycle = ScriptedCycle([KeyError("boom"), KeyError("boom"), 0])
|
||||
stats = self.run_loop(cycle, 3)
|
||||
self.assertEqual(stats.errors_other, 2)
|
||||
self.assertEqual(self.clock.sleeps, [5, 10, 30])
|
||||
self.assertIn("[CYCLE_FAIL] kind=unexpected", self.out.getvalue())
|
||||
self.assertIn("Traceback", self.err.getvalue())
|
||||
self.assertIn("KeyError: 'boom'", self.err.getvalue())
|
||||
|
||||
def test_no_busy_loop_when_ready_sets_keep_failing(self):
|
||||
def progress(stats):
|
||||
stats.image_done(True)
|
||||
return 1
|
||||
|
||||
cycle = ScriptedCycle([2, progress, 0])
|
||||
self.run_loop(cycle, 3)
|
||||
# 2 Saetze ohne Erfolg -> Poll-Pause; Fortschritt -> sofort weiter; leer -> Poll-Pause
|
||||
self.assertEqual(self.clock.sleeps, [30, 30])
|
||||
|
||||
def test_heartbeat_log_and_file(self):
|
||||
stats = self.run_loop(ScriptedCycle([0] * 12), 12)
|
||||
beats = [line for line in self.out.getvalue().splitlines() if line.startswith("[HEARTBEAT]")]
|
||||
self.assertEqual(len(beats), 1) # Zyklus 11 laeuft bei +300 s -> genau eine Zeile
|
||||
self.assertIn("cycles=11 cycles_failed=0", beats[0])
|
||||
self.assertIn("last_success_age=0s", beats[0])
|
||||
with open(self.cfg.heartbeat_file, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
self.assertEqual(data["cycles"], stats.cycles)
|
||||
self.assertEqual(data["errors_transient"], 0)
|
||||
|
||||
|
||||
class RunOnceWithFakeS3Test(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_listing_failure_propagates_to_loop_which_survives(self):
|
||||
s3 = FakeS3(list_error=s3_unavailable())
|
||||
with self.assertRaises(ClientError):
|
||||
main.run_once(self.cfg, None, s3c=s3)
|
||||
|
||||
clock = FakeClock()
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
stats = main.poll_forever(
|
||||
self.cfg,
|
||||
cycle=lambda cfg, st: main.run_once(cfg, st, s3c=s3),
|
||||
init_schema=lambda cfg: None,
|
||||
sleep=clock.sleep,
|
||||
clock=clock.time,
|
||||
max_iterations=3,
|
||||
)
|
||||
self.assertEqual(clock.sleeps, [5, 10, 20])
|
||||
self.assertEqual(stats.errors_transient, 3)
|
||||
|
||||
def test_empty_entrance_returns_zero(self):
|
||||
self.assertEqual(main.run_once(self.cfg, None, s3c=FakeS3()), 0)
|
||||
|
||||
|
||||
class ProcessBatchTest(unittest.TestCase):
|
||||
"""Transaktion pro Bild bleibt; bei S3-/DB-Stoerung wird der Rest des Batches nicht angefasst."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
self.status_calls = []
|
||||
self.processed = []
|
||||
patches = [
|
||||
mock.patch.object(main, "pg_connect", lambda cfg: FakeConn()),
|
||||
mock.patch.object(main, "is_completed_job_materialized", lambda cur, s3, cfg, uuid: False),
|
||||
mock.patch.object(main, "job_upsert_new", lambda cur, uuid, *a: None),
|
||||
mock.patch.object(main, "job_get", lambda cur, uuid: {"status": "CLEANED"}),
|
||||
mock.patch.object(
|
||||
main, "job_set_status",
|
||||
lambda cur, uuid, status, **kw: self.status_calls.append((uuid, status, kw.get("err"))),
|
||||
),
|
||||
]
|
||||
for p in patches:
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def batch(self):
|
||||
s3 = FakeS3(entrance_objects(*UUIDS))
|
||||
return main.collect_ready_sets(s3, self.cfg), s3
|
||||
|
||||
def fail_first_with(self, exc):
|
||||
def process_one(cur, s3, cfg, s):
|
||||
self.processed.append(s.uuid)
|
||||
if s.uuid == UUIDS[0]:
|
||||
raise exc
|
||||
return mock.patch.object(main, "process_one", process_one)
|
||||
|
||||
def test_transient_s3_error_marks_image_error_and_aborts_batch(self):
|
||||
sets, s3 = self.batch()
|
||||
stats = main.PollerStats(None)
|
||||
with self.fail_first_with(client_error("ServiceUnavailable", 503, "GetObject")), \
|
||||
contextlib.redirect_stdout(io.StringIO()) as out:
|
||||
with self.assertRaises(ClientError):
|
||||
main._process_ready_batch(self.cfg, sets, s3, stats)
|
||||
self.assertEqual(self.processed, [UUIDS[0]])
|
||||
self.assertEqual([(u, st) for u, st, _ in self.status_calls], [(UUIDS[0], "ERROR")])
|
||||
self.assertIn(f"[RESULT] uuid={UUIDS[0]} result=failure status=ERROR", out.getvalue())
|
||||
self.assertEqual((stats.images_ok, stats.images_failed), (0, 1))
|
||||
|
||||
def test_bad_image_does_not_stop_batch(self):
|
||||
sets, s3 = self.batch()
|
||||
stats = main.PollerStats(None)
|
||||
with self.fail_first_with(ValueError("metadata.servertime missing")), \
|
||||
contextlib.redirect_stdout(io.StringIO()):
|
||||
main._process_ready_batch(self.cfg, sets, s3, stats)
|
||||
self.assertEqual(self.processed, UUIDS)
|
||||
self.assertEqual(self.status_calls, [(UUIDS[0], "ERROR", "metadata.servertime missing")])
|
||||
self.assertEqual((stats.images_ok, stats.images_failed), (2, 1))
|
||||
|
||||
def test_db_down_propagates_as_transient(self):
|
||||
sets, s3 = self.batch()
|
||||
|
||||
def no_db(cfg):
|
||||
raise psycopg.OperationalError("connection timeout expired")
|
||||
|
||||
with mock.patch.object(main, "pg_connect", no_db):
|
||||
with self.assertRaises(psycopg.OperationalError) as ctx:
|
||||
main._process_ready_batch(self.cfg, sets, s3)
|
||||
self.assertTrue(main.is_transient_error(ctx.exception))
|
||||
|
||||
def test_run_once_parallel_raises_when_a_worker_hits_outage(self):
|
||||
cfg = make_cfg(self.tmp.name, parallel_workers=3)
|
||||
s3 = FakeS3(entrance_objects(*UUIDS))
|
||||
with self.fail_first_with(s3_unavailable()), contextlib.redirect_stdout(io.StringIO()):
|
||||
with self.assertRaises(ClientError):
|
||||
main.run_once(cfg, main.PollerStats(None), s3c=s3)
|
||||
self.assertEqual(sorted(self.processed), UUIDS) # die anderen Worker laufen zu Ende
|
||||
|
||||
|
||||
class S3ClientConfigTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cfg = make_cfg(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_poller_client_has_explicit_timeouts_and_retries(self):
|
||||
conf = main.S3(self.cfg, main.s3_client_config(self.cfg)).client.meta.config
|
||||
self.assertEqual(conf.connect_timeout, 10)
|
||||
self.assertEqual(conf.read_timeout, 30)
|
||||
self.assertEqual(conf.retries.get("mode"), "standard")
|
||||
self.assertEqual(conf.retries.get("total_max_attempts"), 4)
|
||||
self.assertNotIn("max_attempts", conf.retries)
|
||||
self.assertEqual(conf.signature_version, "s3v4")
|
||||
self.assertEqual(conf.s3.get("addressing_style"), "path")
|
||||
|
||||
def test_default_client_unchanged_for_gundi_path(self):
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
# exakt wie vor der Aenderung gebaut
|
||||
before = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=self.cfg.s3_endpoint,
|
||||
aws_access_key_id=self.cfg.s3_access_key,
|
||||
aws_secret_access_key=self.cfg.s3_secret_key,
|
||||
config=Config(max_pool_connections=self.cfg.s3_max_pool_connections),
|
||||
).meta.config
|
||||
now = main.S3(self.cfg).client.meta.config
|
||||
for attr in ("connect_timeout", "read_timeout", "retries", "max_pool_connections",
|
||||
"signature_version", "s3", "region_name"):
|
||||
with self.subTest(attr=attr):
|
||||
self.assertEqual(getattr(now, attr), getattr(before, attr))
|
||||
self.assertEqual(now.retries, {"mode": "legacy"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bestandsaufnahme des S3-Eingangs (icu/entrance/) - rein lesend.
|
||||
|
||||
Gruppiert alle Objekte unter dem Eingangspraefix nach Bild-UUID und ordnet sie ein:
|
||||
|
||||
complete JPG + Metadaten-JSON (+ optional _categories.json) - verarbeitbar
|
||||
jpg_only JPG ohne Metadaten-JSON (evtl. mit _categories.json)
|
||||
json_only Metadaten-JSON ohne JPG (evtl. mit _categories.json)
|
||||
cat_only nur _categories.json
|
||||
foreign Name passt zu keinem ICU-Muster
|
||||
|
||||
Alter = Alter des juengsten Objekts der Gruppe (konservativ).
|
||||
|
||||
python -m tools.entrance_inventory # Zusammenfassung
|
||||
python -m tools.entrance_inventory --db # + Status aus remote_cam.import_job
|
||||
python -m tools.entrance_inventory --list jpg_only --min-age-days 7
|
||||
python -m tools.entrance_inventory --db --csv /tmp/entrance.csv
|
||||
|
||||
Liest Zugangsdaten wie der Poller (CONFIG_YAML + .env). Schreibt/loescht nichts.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from main import CAT_RE, S3, UUID_RE, load_config
|
||||
|
||||
TYPES = ["complete", "jpg_only", "json_only", "cat_only", "foreign"]
|
||||
AGE_BUCKETS = [
|
||||
("<1h", 3600),
|
||||
("1h-1d", 86400),
|
||||
("1-7d", 7 * 86400),
|
||||
("7-30d", 30 * 86400),
|
||||
(">30d", None),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Group:
|
||||
uuid: Optional[str]
|
||||
parts: dict = field(default_factory=dict) # jpg/meta/cat -> (key, last_modified, size)
|
||||
|
||||
@property
|
||||
def kind(self) -> str:
|
||||
if self.uuid is None:
|
||||
return "foreign"
|
||||
has_jpg, has_meta = "jpg" in self.parts, "meta" in self.parts
|
||||
if has_jpg and has_meta:
|
||||
return "complete"
|
||||
if has_jpg:
|
||||
return "jpg_only"
|
||||
if has_meta:
|
||||
return "json_only"
|
||||
return "cat_only"
|
||||
|
||||
@property
|
||||
def newest(self) -> datetime:
|
||||
return max(p[1] for p in self.parts.values())
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return sum(p[2] for p in self.parts.values())
|
||||
|
||||
@property
|
||||
def keys(self) -> list[str]:
|
||||
return [p[0] for p in self.parts.values()]
|
||||
|
||||
|
||||
def age_bucket(age_s: float) -> str:
|
||||
for name, limit in AGE_BUCKETS:
|
||||
if limit is None or age_s < limit:
|
||||
return name
|
||||
return AGE_BUCKETS[-1][0]
|
||||
|
||||
|
||||
def collect(s3: S3) -> list[Group]:
|
||||
groups: dict[str, Group] = {}
|
||||
foreign: list[Group] = []
|
||||
for obj in s3.list_entrance_objects():
|
||||
key = obj["Key"]
|
||||
name = key.split("/")[-1]
|
||||
entry = (key, obj["LastModified"], obj.get("Size", 0))
|
||||
m = UUID_RE.match(name)
|
||||
if m:
|
||||
typ = "jpg" if m.group(2) == "jpg" else "meta"
|
||||
groups.setdefault(m.group(1), Group(m.group(1))).parts[typ] = entry
|
||||
continue
|
||||
m = CAT_RE.match(name)
|
||||
if m:
|
||||
groups.setdefault(m.group(1), Group(m.group(1))).parts["cat"] = entry
|
||||
continue
|
||||
foreign.append(Group(None, {"other": entry}))
|
||||
return list(groups.values()) + foreign
|
||||
|
||||
|
||||
def load_job_status(dsn: str, uuids: list[str]) -> dict[str, tuple]:
|
||||
import psycopg
|
||||
|
||||
out: dict[str, tuple] = {}
|
||||
with psycopg.connect(dsn, connect_timeout=10) as db, db.cursor() as cur:
|
||||
cur.execute("SET TRANSACTION READ ONLY")
|
||||
for i in range(0, len(uuids), 5000):
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT image_uuid, status, attempts, updated_ts, left(last_error, 200)
|
||||
FROM remote_cam.import_job
|
||||
WHERE image_uuid = ANY(%s)
|
||||
""",
|
||||
(uuids[i:i + 5000],),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
out[row[0]] = row[1:]
|
||||
return out
|
||||
|
||||
|
||||
def fmt_ts(dt: Optional[datetime]) -> str:
|
||||
return dt.strftime("%Y-%m-%d %H:%M") if dt else "-"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--config", default=None, help="Pfad zu config.yaml (Default: $CONFIG_YAML bzw. /app/config.yaml)")
|
||||
ap.add_argument("--db", action="store_true", help="Status je UUID aus remote_cam.import_job lesen")
|
||||
ap.add_argument("--list", choices=TYPES, help="Keys dieses Typs ausgeben")
|
||||
ap.add_argument("--min-age-days", type=float, default=0.0, help="nur Gruppen, deren juengstes Objekt aelter ist")
|
||||
ap.add_argument("--csv", help="alle Gruppen als CSV schreiben")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config or os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
now = datetime.now(timezone.utc)
|
||||
groups = collect(S3(cfg))
|
||||
|
||||
status: dict[str, tuple] = {}
|
||||
if args.db:
|
||||
status = load_job_status(cfg.pg_dsn, [g.uuid for g in groups if g.uuid])
|
||||
|
||||
min_age_s = args.min_age_days * 86400
|
||||
selected = [g for g in groups if (now - g.newest).total_seconds() >= min_age_s]
|
||||
|
||||
# --- Zusammenfassung: Typ x Altersklasse
|
||||
by_kind: dict[str, list[Group]] = defaultdict(list)
|
||||
for g in selected:
|
||||
by_kind[g.kind].append(g)
|
||||
|
||||
bucket_names = [b[0] for b in AGE_BUCKETS]
|
||||
print(f"Eingang s3://{cfg.s3_bucket}/{cfg.entrance_prefix} Stand {now:%Y-%m-%d %H:%M} UTC"
|
||||
f" ({sum(len(g.parts) for g in selected)} Objekte, {len(selected)} Gruppen"
|
||||
+ (f", juengstes Objekt >= {args.min_age_days:g} Tage" if min_age_s else "") + ")")
|
||||
print()
|
||||
header = f"{'Typ':<10} {'Gruppen':>8} {'MB':>9} " + " ".join(f"{b:>6}" for b in bucket_names) + " aeltestes juengstes"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for kind in TYPES:
|
||||
gs = by_kind.get(kind, [])
|
||||
if not gs:
|
||||
continue
|
||||
buckets = Counter(age_bucket((now - g.newest).total_seconds()) for g in gs)
|
||||
mb = sum(g.size for g in gs) / 1e6
|
||||
print(f"{kind:<10} {len(gs):>8} {mb:>9.1f} " + " ".join(f"{buckets.get(b, 0):>6}" for b in bucket_names)
|
||||
+ f" {fmt_ts(min(g.newest for g in gs)):<18} {fmt_ts(max(g.newest for g in gs))}")
|
||||
|
||||
with_cat = Counter(g.kind for g in selected if "cat" in g.parts and g.kind != "cat_only")
|
||||
if with_cat:
|
||||
print("\ndavon mit _categories.json: " + ", ".join(f"{k}={v}" for k, v in sorted(with_cat.items())))
|
||||
|
||||
if args.db:
|
||||
print("\nimport_job-Status je Typ:")
|
||||
for kind in TYPES[:-1]:
|
||||
gs = by_kind.get(kind, [])
|
||||
if gs:
|
||||
c = Counter((status.get(g.uuid) or ("kein Job",))[0] for g in gs)
|
||||
print(f" {kind:<10} " + ", ".join(f"{k}={v}" for k, v in c.most_common()))
|
||||
|
||||
# Fremde Objekte immer einzeln zeigen - die gehoeren nicht in den Eingang.
|
||||
if by_kind.get("foreign"):
|
||||
print("\nFremde Objekte:")
|
||||
for g in sorted(by_kind["foreign"], key=lambda x: x.newest):
|
||||
key, lm, size = g.parts["other"]
|
||||
print(f" {fmt_ts(lm)} {size:>10} B {key}")
|
||||
|
||||
if args.list:
|
||||
print(f"\n# Keys vom Typ {args.list}:")
|
||||
for g in sorted(by_kind.get(args.list, []), key=lambda x: x.newest):
|
||||
for key in sorted(g.keys):
|
||||
print(key)
|
||||
|
||||
if args.csv:
|
||||
with open(args.csv, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["kind", "uuid", "has_jpg", "has_meta", "has_cat", "newest_utc", "age_days", "bytes",
|
||||
"job_status", "job_attempts", "job_updated_utc", "job_last_error", "keys"])
|
||||
for g in sorted(selected, key=lambda x: (x.kind, x.newest)):
|
||||
st = status.get(g.uuid) if g.uuid else None
|
||||
w.writerow([
|
||||
g.kind, g.uuid or "", "jpg" in g.parts, "meta" in g.parts, "cat" in g.parts,
|
||||
g.newest.isoformat(), round((now - g.newest).total_seconds() / 86400, 2), g.size,
|
||||
st[0] if st else "", st[1] if st else "", st[2].isoformat() if st and st[2] else "",
|
||||
st[3] if st else "", " ".join(sorted(g.keys)),
|
||||
])
|
||||
print(f"\nCSV geschrieben: {args.csv}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet.
|
||||
|
||||
python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen
|
||||
python -m tools.gundi_requeue --apply # setzt zurueck
|
||||
python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778
|
||||
|
||||
Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue
|
||||
|
||||
- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3
|
||||
(sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen.
|
||||
- image ocr_state FAIL -> PENDING (ocr_attempts=0)
|
||||
Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from gundi_pipeline import connect # noqa: E402
|
||||
from main import load_config # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)")
|
||||
ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)")
|
||||
ap.add_argument("--imei", help="nur diese IMEI (source)")
|
||||
ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL"
|
||||
image_where = "source = 'gundi' AND ocr_state = 'FAIL'"
|
||||
params_inbox: list = []
|
||||
params_image: list = []
|
||||
if args.since:
|
||||
inbox_where += " AND received_at >= %s"
|
||||
image_where += " AND servertime >= %s"
|
||||
params_inbox.append(args.since)
|
||||
params_image.append(args.since)
|
||||
if args.imei:
|
||||
inbox_where += " AND source = %s"
|
||||
image_where += " AND imei = %s"
|
||||
params_inbox.append(args.imei)
|
||||
params_image.append(args.imei)
|
||||
|
||||
with connect(cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox)
|
||||
n_inbox = cur.fetchone()[0]
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image)
|
||||
n_image = cur.fetchone()[0]
|
||||
print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}")
|
||||
print(f"image OCR FAIL: {n_image}")
|
||||
if not args.apply:
|
||||
print("Trockenlauf - mit --apply zuruecksetzen.")
|
||||
return 0
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {inbox_where}
|
||||
""",
|
||||
params_inbox,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege")
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {image_where}
|
||||
""",
|
||||
params_image,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege")
|
||||
print("Der Sweep uebernimmt sie in der naechsten Runde.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Spielt die Beispiel-Payloads aus der ICU-Doku gegen den Gundi-Receiver.
|
||||
|
||||
python -m tools.gundi_smoke_test --url http://127.0.0.1:8090 --jpeg bild.jpg
|
||||
|
||||
Ohne --jpeg wird ein synthetisches Bild mit Zeitstempel-Overlay erzeugt (braucht Pillow).
|
||||
Standard-IMEI ist eine Test-IMEI, damit echte Kamerazeitreihen unberuehrt bleiben.
|
||||
"""
|
||||
import argparse
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import urlparse
|
||||
|
||||
TEST_IMEI = "999000000000001"
|
||||
|
||||
|
||||
def request(url: str, method: str, path: str, *, body: bytes = b"", headers: dict | None = None):
|
||||
u = urlparse(url)
|
||||
cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
|
||||
conn = cls(u.hostname, u.port, timeout=30)
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=headers or {})
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
return resp.status, dict(resp.getheaders()), data
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def post_json(url, path, key, payload):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key is not None:
|
||||
headers["apikey"] = key
|
||||
return request(url, "POST", path, body=json.dumps(payload).encode(), headers=headers)
|
||||
|
||||
|
||||
def post_file(url, path, key, jpg: bytes):
|
||||
boundary = f"----smoke-{uuid.uuid4().hex}"
|
||||
body = (
|
||||
f"--{boundary}\r\n"
|
||||
f'Content-Disposition: form-data; name="file1"; filename="../../evil.jpg"\r\n'
|
||||
f"Content-Type: image/jpeg\r\n\r\n"
|
||||
).encode() + jpg + f"\r\n--{boundary}--\r\n".encode()
|
||||
return request(url, "POST", path, body=body, headers={
|
||||
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
||||
"apikey": key,
|
||||
})
|
||||
|
||||
|
||||
def synthetic_jpeg(ts: datetime) -> bytes:
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
w, h = 1920, 1080
|
||||
im = Image.new("RGB", (w, h), (70, 90, 70))
|
||||
draw = ImageDraw.Draw(im)
|
||||
bar = int(h * 0.05)
|
||||
draw.rectangle((0, h - bar, w, h), fill=(0, 0, 0))
|
||||
font = ImageFont.load_default(size=int(bar * 0.8))
|
||||
text = ts.strftime("%H:%M:%S %d-%m-%Y")
|
||||
tw = draw.textlength(text, font=font)
|
||||
draw.text((w - tw - 20, h - bar + bar * 0.08), text, fill=(255, 255, 255), font=font)
|
||||
out = io.BytesIO()
|
||||
im.save(out, format="JPEG", quality=90)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default="http://127.0.0.1:8090")
|
||||
ap.add_argument("--key", default=os.environ.get("GUNDI_API_KEY"))
|
||||
ap.add_argument("--imei", default=TEST_IMEI)
|
||||
ap.add_argument("--jpeg", help="Pfad zu einem echten Kamerabild")
|
||||
args = ap.parse_args()
|
||||
if not args.key:
|
||||
print("--key oder GUNDI_API_KEY fehlt", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
failures = 0
|
||||
|
||||
def check(label: str, ok: bool, detail: str = "") -> None:
|
||||
nonlocal failures
|
||||
failures += 0 if ok else 1
|
||||
print(f"[{'OK ' if ok else 'FAIL'}] {label}{(' - ' + detail) if detail else ''}")
|
||||
|
||||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
status_payload = {
|
||||
"source": args.imei,
|
||||
"event_type": "camera_status_report",
|
||||
"title": "Camera Status report - Smoke Test",
|
||||
"recorded_at": (now - timedelta(minutes=10)).isoformat().replace("+00:00", "Z"),
|
||||
"location": {"lat": 0.0, "lon": 0.0},
|
||||
"event_details": {
|
||||
"imei": args.imei, "iccid": "8900000000000000000", "init_key": "000000000000",
|
||||
"camera_name": "Smoke Test", "camera_model": None, "firmware_version": "1.4.2",
|
||||
"mobile_country_code": "232", "mobile_network_code": "01", "location_area_code": "1234",
|
||||
"cell_id": "567890", "customer_timezone": "Europe/Vienna", "battery_status": 87,
|
||||
"signal_strength": 3, "temperature": -4, "coordinate_switch": False, "work_period": True,
|
||||
"image_format": "F4_3", "image_night": "MIN", "work_start": "06:00", "work_end": "21:00",
|
||||
"thumbnail_size": "720p", "pir_interval": "30", "time_scan": "120", "camera_language": "en",
|
||||
},
|
||||
}
|
||||
trap_payload = {
|
||||
"source": args.imei,
|
||||
"event_type": "cameratrap_rep",
|
||||
"title": "Camera trap image - Smoke Test",
|
||||
"recorded_at": now.isoformat().replace("+00:00", "Z"),
|
||||
"location": {"lat": 47.8095, "lon": 13.0550},
|
||||
"event_details": {
|
||||
"cameratraprep_camera-name": "Smoke Test",
|
||||
"cameratraprep_camera-make": "M350GW",
|
||||
"detections": [
|
||||
{"category": "deer", "confidence_percent": 96.2, "x_px": 0, "y_px": 0, "width_px": 0, "height_px": 0},
|
||||
{"category": "human", "confidence_percent": 3.1, "x_px": 15, "y_px": 17, "width_px": 100, "height_px": 200},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
code, _, _ = request(args.url, "GET", "/healthz")
|
||||
check("healthz", code == 200, f"HTTP {code}")
|
||||
|
||||
code, _, _ = post_json(args.url, "/v2/events/", None, status_payload)
|
||||
check("ohne apikey -> 401", code == 401, f"HTTP {code}")
|
||||
code, _, _ = post_json(args.url, "/v2/events/", "falsch", status_payload)
|
||||
check("falscher apikey -> 401", code == 401, f"HTTP {code}")
|
||||
|
||||
code, headers, _ = post_json(args.url, "/v2/events", args.key, status_payload)
|
||||
check("ohne Slash -> 308", code == 308 and headers.get("Location") == "/v2/events/",
|
||||
f"HTTP {code} Location={headers.get('Location')}")
|
||||
|
||||
code, _, data = post_json(args.url, "/v2/events/", args.key, {"event_type": "cameratrap_rep"})
|
||||
check("unvollstaendiges Event -> 400", code == 400, f"HTTP {code} {data[:120]!r}")
|
||||
|
||||
code, _, data = post_json(args.url, "/v2/events/", args.key, status_payload)
|
||||
status_id = json.loads(data).get("object_id") if code == 200 else None
|
||||
check("camera_status_report", code == 200 and bool(status_id), f"HTTP {code} object_id={status_id}")
|
||||
code, _, _ = post_json(args.url, "/v2/events/", args.key, status_payload)
|
||||
check("camera_status_report erneut (idempotent in camera_status)", code == 200, f"HTTP {code}")
|
||||
|
||||
code, _, data = post_json(args.url, "/v2/events/", args.key, trap_payload)
|
||||
object_id = json.loads(data).get("object_id") if code == 200 else None
|
||||
check("cameratrap_rep", code == 200 and bool(object_id), f"HTTP {code} object_id={object_id}")
|
||||
|
||||
if args.jpeg:
|
||||
with open(args.jpeg, "rb") as f:
|
||||
jpg = f.read()
|
||||
else:
|
||||
jpg = synthetic_jpeg(now.astimezone(timezone(timedelta(hours=2))))
|
||||
|
||||
code, _, _ = post_file(args.url, f"/v2/events/{uuid.uuid4()}/attachments/", args.key, jpg)
|
||||
check("Attachment auf unbekannte object_id -> 404", code == 404, f"HTTP {code}")
|
||||
code, _, _ = post_file(args.url, "/v2/events/..%2F..%2Fx/attachments/", args.key, jpg)
|
||||
check("Attachment auf ungueltige object_id -> 404", code == 404, f"HTTP {code}")
|
||||
code, _, _ = post_file(args.url, f"/v2/events/{status_id}/attachments/", args.key, jpg)
|
||||
check("Attachment auf Status-Event -> 404", code == 404, f"HTTP {code}")
|
||||
|
||||
if object_id:
|
||||
code, headers, _ = post_file(args.url, f"/v2/events/{object_id}/attachments", args.key, jpg)
|
||||
check("Attachment ohne Slash -> 308", code == 308, f"HTTP {code}")
|
||||
code, _, data = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg)
|
||||
check("Attachment", code == 200, f"HTTP {code} {data[:120]!r}")
|
||||
code, _, _ = post_file(args.url, f"/v2/events/{object_id}/attachments/", args.key, jpg)
|
||||
check("Attachment erneut (duplicate, 200)", code == 200, f"HTTP {code}")
|
||||
|
||||
print(f"\nobject_id cameratrap_rep: {object_id}")
|
||||
print(f"object_id status_report: {status_id}")
|
||||
print(f"{failures} Fehler")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migriert remote_cam.metadata/category batchweise ins Fachmodell (image, camera_status, detection).
|
||||
|
||||
python -m tools.migrate_legacy # PG_DSN aus der Umgebung
|
||||
python -m tools.migrate_legacy --dsn postgresql://... --batch 20000
|
||||
python -m tools.migrate_legacy --verify-only
|
||||
python -m tools.migrate_legacy --plan # nur lesen: was wuerde migriert/entfernt
|
||||
python -m tools.migrate_legacy --dedupe --dry-run
|
||||
python -m tools.migrate_legacy --dedupe # Gundi-Zwillinge entfernen (schema/003)
|
||||
|
||||
Wiederholbar. Jeder Batch ist eine eigene Transaktion, --dedupe laeuft in einer.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
|
||||
SCHEMA_LOCK_KEY = 7_311_000_001
|
||||
|
||||
|
||||
def load_steps(filename: str = "002_migrate_legacy.sql") -> list[tuple[str, str]]:
|
||||
sql = (SCHEMA_DIR / filename).read_text(encoding="utf-8")
|
||||
parts = re.split(r"^-- step: (\w+)\s*$", sql, flags=re.MULTILINE)
|
||||
return [(parts[i], parts[i + 1].strip()) for i in range(1, len(parts), 2)]
|
||||
|
||||
|
||||
def apply_schema(conn: psycopg.Connection) -> None:
|
||||
sql = (SCHEMA_DIR / "001_domain_model.sql").read_text(encoding="utf-8")
|
||||
with conn.transaction(), conn.cursor() as cur:
|
||||
cur.execute("SELECT pg_advisory_xact_lock(%s)", (SCHEMA_LOCK_KEY,))
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
def migrate(conn: psycopg.Connection, batch_size: int) -> None:
|
||||
steps = load_steps()
|
||||
totals = {name: 0 for name, _ in steps}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT count(*) FROM remote_cam.metadata")
|
||||
total_rows = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"CREATE TEMP TABLE IF NOT EXISTS _mig_batch (metadata_uuid text PRIMARY KEY) ON COMMIT DELETE ROWS"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
last = ""
|
||||
seen = 0
|
||||
t0 = time.perf_counter()
|
||||
while True:
|
||||
with conn.transaction(), conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO _mig_batch
|
||||
SELECT metadata_uuid FROM remote_cam.metadata
|
||||
WHERE metadata_uuid > %s ORDER BY metadata_uuid LIMIT %s
|
||||
""",
|
||||
(last, batch_size),
|
||||
)
|
||||
n = cur.rowcount
|
||||
if n == 0:
|
||||
break
|
||||
cur.execute("SELECT max(metadata_uuid) FROM _mig_batch")
|
||||
last = cur.fetchone()[0]
|
||||
added = []
|
||||
for name, sql in steps:
|
||||
cur.execute(sql)
|
||||
totals[name] += cur.rowcount
|
||||
added.append(f"{name}+{cur.rowcount}")
|
||||
seen += n
|
||||
rate = seen / max(time.perf_counter() - t0, 1e-6)
|
||||
print(f" {seen:>8}/{total_rows} {' '.join(added)} ({rate:.0f} Zeilen/s)", flush=True)
|
||||
|
||||
print("Neu angelegt: " + ", ".join(f"{k}={v}" for k, v in totals.items()), flush=True)
|
||||
|
||||
|
||||
# Welche Zeilen aus remote_cam.category (Alias c) zu detection werden -- wie in 002.
|
||||
DETECTION_FILTER = (
|
||||
"c.category IS NOT NULL AND c.category NOT IN ('', 'none') AND (c.score >= 0.5 "
|
||||
"OR (c.category IN ('human', 'machine') AND NOT EXISTS (SELECT 1 FROM remote_cam.category c2 "
|
||||
"WHERE c2.image_uuid = c.image_uuid "
|
||||
"AND (c2.score > c.score OR (c2.score = c.score AND c2.category < c.category)))))"
|
||||
)
|
||||
|
||||
# Lesende Vorschau; die Bedingungen spiegeln 002 (detection-Filter) und 003 (Zwillinge).
|
||||
PLAN_QUERIES = [
|
||||
("image(entrance) neu",
|
||||
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM remote_cam.image i WHERE i.image_uuid = m.metadata_uuid::uuid)"),
|
||||
("camera_status(entrance) neu",
|
||||
"SELECT count(*) FROM remote_cam.metadata m WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM remote_cam.camera_status s WHERE s.source = 'entrance' "
|
||||
"AND s.source_ref = m.metadata_uuid)"),
|
||||
("detection(entrance) neu",
|
||||
"SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m ON m.metadata_uuid = c.image_uuid "
|
||||
"WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL AND " + DETECTION_FILTER + " "
|
||||
"AND NOT EXISTS (SELECT 1 FROM remote_cam.detection d WHERE d.image_uuid = m.metadata_uuid::uuid "
|
||||
"AND d.source = 'entrance')"),
|
||||
("Gundi-Zwillinge (nach Migration zu entfernen)",
|
||||
"""
|
||||
WITH pairs AS (
|
||||
SELECT g.image_uuid AS gundi_uuid, m.metadata_uuid AS legacy_uuid
|
||||
FROM remote_cam.image g
|
||||
JOIN remote_cam.metadata m
|
||||
ON m.imei = g.imei AND m."SizeInByte" = g.size_bytes
|
||||
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes'
|
||||
WHERE g.source = 'gundi')
|
||||
SELECT count(*) FROM (
|
||||
SELECT count(*) OVER (PARTITION BY gundi_uuid) AS n_g, count(*) OVER (PARTITION BY legacy_uuid) AS n_l
|
||||
FROM pairs) x
|
||||
WHERE n_g = 1 AND n_l = 1
|
||||
"""),
|
||||
("Gundi-Bilder ohne Zwilling (bleiben)",
|
||||
"""
|
||||
SELECT count(*) FROM remote_cam.image g WHERE g.source = 'gundi' AND NOT EXISTS (
|
||||
SELECT 1 FROM remote_cam.metadata m
|
||||
WHERE m.imei = g.imei AND m."SizeInByte" = g.size_bytes
|
||||
AND m.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes')
|
||||
"""),
|
||||
]
|
||||
|
||||
|
||||
def plan(conn: psycopg.Connection) -> None:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SET TRANSACTION READ ONLY")
|
||||
for label, sql in PLAN_QUERIES:
|
||||
cur.execute(sql)
|
||||
print(f" {label}: {cur.fetchone()[0]}", flush=True)
|
||||
conn.rollback()
|
||||
|
||||
|
||||
def dedupe(conn: psycopg.Connection, dry_run: bool) -> None:
|
||||
steps = dict(load_steps("003_dedupe_gundi_twins.sql"))
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(steps["twins"])
|
||||
cur.execute("SELECT count(*) FROM _twins")
|
||||
n = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"SELECT min(i.servertime), max(i.servertime) FROM _twins t "
|
||||
"JOIN remote_cam.image i ON i.image_uuid = t.gundi_uuid"
|
||||
)
|
||||
lo, hi = cur.fetchone()
|
||||
print(f" Gundi-Zwillinge: {n} (servertime {lo} .. {hi})", flush=True)
|
||||
if dry_run:
|
||||
conn.rollback()
|
||||
print(" Trockenlauf: nichts geaendert.", flush=True)
|
||||
return
|
||||
cur.execute(steps["archive"])
|
||||
archived = cur.rowcount
|
||||
cur.execute(steps["delete"])
|
||||
deleted = cur.rowcount
|
||||
if deleted != n:
|
||||
conn.rollback()
|
||||
raise RuntimeError(f"geloescht={deleted} != Zwillinge={n}; zurueckgerollt")
|
||||
conn.commit()
|
||||
print(f" archiviert={archived} geloescht={deleted}", flush=True)
|
||||
|
||||
|
||||
def verify(conn: psycopg.Connection) -> bool:
|
||||
checks = [
|
||||
("metadata == image(entrance)",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
||||
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance')"),
|
||||
("metadata == camera_status(entrance)",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.metadata), "
|
||||
"(SELECT count(*) FROM remote_cam.camera_status WHERE source = 'entrance')"),
|
||||
("category (Score >= 0.5 bzw. Top human/machine, ohne ''/none) == detection(entrance)",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.category c JOIN remote_cam.metadata m "
|
||||
"ON m.metadata_uuid = c.image_uuid WHERE m.imei IS NOT NULL AND m.servertime IS NOT NULL "
|
||||
"AND " + DETECTION_FILTER + "), "
|
||||
"(SELECT count(*) FROM remote_cam.detection WHERE source = 'entrance')"),
|
||||
("keine Gundi-Bilder mit Legacy-Zwilling",
|
||||
"SELECT 0, count(*) FROM remote_cam.image g JOIN remote_cam.image e "
|
||||
"ON e.source = 'entrance' AND e.imei = g.imei AND e.size_bytes = g.size_bytes "
|
||||
"AND e.servertime BETWEEN g.servertime - interval '10 minutes' AND g.servertime + interval '10 minutes' "
|
||||
"WHERE g.source = 'gundi'"),
|
||||
("metadata ohne ocr_ts == image FAIL",
|
||||
"SELECT (SELECT count(*) FROM remote_cam.metadata WHERE ocr_ts IS NULL), "
|
||||
"(SELECT count(*) FROM remote_cam.image WHERE source = 'entrance' AND ocr_state = 'FAIL')"),
|
||||
("keine 0/0-Koordinaten (image, camera_status)",
|
||||
"SELECT 0, (SELECT count(*) FROM remote_cam.image WHERE lat = 0 AND lon = 0) + "
|
||||
"(SELECT count(*) FROM remote_cam.camera_status WHERE lat = 0 AND lon = 0)"),
|
||||
("Stichprobe: image_with_status == metadata (200 Zufallszeilen, Abweichungen)",
|
||||
"""
|
||||
SELECT 0, count(*) FROM (
|
||||
SELECT m.* FROM remote_cam.metadata m TABLESAMPLE SYSTEM (1) LIMIT 200
|
||||
) m
|
||||
JOIN remote_cam.image_with_status v ON v.image_uuid = m.metadata_uuid::uuid
|
||||
WHERE v.status_reported_at IS DISTINCT FROM m.servertime
|
||||
OR v.battery_pct IS DISTINCT FROM m."BatteryStatus"
|
||||
OR v.firmware_version IS DISTINCT FROM m."FirmwareVersion"
|
||||
OR v.status_camera_name IS DISTINCT FROM m."CameraName"
|
||||
OR v.signal_strength IS DISTINCT FROM m."SignalStrength"
|
||||
OR v.temperature_c IS DISTINCT FROM m."Temperature"
|
||||
OR v.customer_timezone IS DISTINCT FROM m."CustomerTimezone"
|
||||
OR v.ocr_ts IS DISTINCT FROM m.ocr_ts
|
||||
"""),
|
||||
]
|
||||
ok_all = True
|
||||
with conn.cursor() as cur:
|
||||
for label, sql in checks:
|
||||
cur.execute(sql)
|
||||
expected, actual = cur.fetchone()
|
||||
ok = expected == actual
|
||||
ok_all &= ok
|
||||
print(f"[{'OK ' if ok else 'FAIL'}] {label}: erwartet={expected} ist={actual}", flush=True)
|
||||
conn.rollback()
|
||||
return ok_all
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dsn", default=os.environ.get("PG_DSN"))
|
||||
ap.add_argument("--batch", type=int, default=20000)
|
||||
ap.add_argument("--verify-only", action="store_true")
|
||||
ap.add_argument("--plan", action="store_true", help="nur lesen: Vorschau der Mengen")
|
||||
ap.add_argument("--dedupe", action="store_true", help="Gundi-Zwillinge entfernen (schema/003)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="mit --dedupe: nur zaehlen")
|
||||
args = ap.parse_args()
|
||||
if not args.dsn:
|
||||
print("--dsn oder PG_DSN fehlt", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
with psycopg.connect(args.dsn, connect_timeout=10) as conn:
|
||||
if args.plan:
|
||||
print("Vorschau (nur lesend):", flush=True)
|
||||
plan(conn)
|
||||
return 0
|
||||
if args.dedupe:
|
||||
if not args.dry_run:
|
||||
apply_schema(conn)
|
||||
print("Gundi-Zwillinge:", flush=True)
|
||||
dedupe(conn, args.dry_run)
|
||||
return 0
|
||||
if not args.verify_only:
|
||||
apply_schema(conn)
|
||||
print("Schema angewendet. Migriere ...", flush=True)
|
||||
migrate(conn, args.batch)
|
||||
print("Verifikation:", flush=True)
|
||||
return 0 if verify(conn) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
# Live-Ueberwachung des Gundi-Push-Empfaengers: kommen ueberhaupt POSTs von ICU an?
|
||||
#
|
||||
# Verfolgt zwei Quellen parallel und faerbt die relevanten Zeilen ein:
|
||||
# [app] docker logs des Receiver-Containers ([HTTP]-Requestzeilen, [GUNDI]-Events/Anhaenge, Fehler)
|
||||
# [nginx] nginx-Access-Log (meles_dev3), gefiltert auf /v2/events; Fallback sudo, wenn nicht lesbar
|
||||
# [traefik] Access-Log von Traefik (meles2), gefiltert auf /v2/events - zeigt auch Requests, die Traefik
|
||||
# selbst beantwortet und die nie beim Container ankommen (404 = kein Router-Match,
|
||||
# 502/504 = Container down, 308/301 durch http->https-Redirect ...)
|
||||
#
|
||||
# Voraussetzung fuer [traefik]: Access-Log in Traefik aktiviert (--accesslog=true). Liegt er
|
||||
# als Datei im Container (--accesslog.filepath, hier /var/log/access.log), wird per docker exec
|
||||
# mitgelesen; sonst per docker logs.
|
||||
#
|
||||
# Aufruf auf meles2:
|
||||
# tools/watch_gundi.sh # live ab jetzt, inkl. der letzten 10 Minuten
|
||||
# tools/watch_gundi.sh -s 2h # Rueckblick 2 Stunden, dann live
|
||||
# tools/watch_gundi.sh -a # alle Zeilen statt nur POST/Fehler (inkl. /healthz)
|
||||
# tools/watch_gundi.sh -t "" # ohne Traefik-Log
|
||||
# tools/watch_gundi.sh -f /pfad/access.log # Traefik schreibt in eine Datei statt nach stdout
|
||||
#
|
||||
# Optionen:
|
||||
# -n LOGFILE nginx-Access-Log (Default: /var/log/nginx/access.log wenn vorhanden, "" = aus)
|
||||
# -c CONTAINER Receiver-Container (Default: melesicumover-gundi)
|
||||
# -t CONTAINER Traefik-Container (Default: automatisch, erster Container mit "traefik" im Namen)
|
||||
# -f LOGFILE Traefik-Access-Log als Datei statt docker logs
|
||||
# -s SINCE docker-logs --since (Default: 10m)
|
||||
# -a nichts filtern
|
||||
set -u
|
||||
|
||||
CONTAINER="${GUNDI_CONTAINER:-melesicumover-gundi}"
|
||||
NGINX_LOG="${GUNDI_NGINX_LOG-/var/log/nginx/access.log}"
|
||||
TRAEFIK="${GUNDI_TRAEFIK:-auto}"
|
||||
TRAEFIK_FILE=""
|
||||
TRAEFIK_INNER_LOG="${GUNDI_TRAEFIK_LOG:-/var/log/access.log}"
|
||||
SINCE="10m"
|
||||
ALL=0
|
||||
|
||||
while getopts "c:n:t:f:s:ah" opt; do
|
||||
case "$opt" in
|
||||
c) CONTAINER="$OPTARG" ;;
|
||||
n) NGINX_LOG="$OPTARG" ;;
|
||||
t) TRAEFIK="$OPTARG" ;;
|
||||
f) TRAEFIK_FILE="$OPTARG" ;;
|
||||
s) SINCE="$OPTARG" ;;
|
||||
a) ALL=1 ;;
|
||||
*) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! docker inspect "$CONTAINER" >/dev/null 2>&1; then
|
||||
echo "Container '$CONTAINER' nicht gefunden. Laufende Container:" >&2
|
||||
docker ps --format ' {{.Names}}\t{{.Status}}' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
state=$(docker inspect -f '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}} started={{.State.StartedAt}}' "$CONTAINER")
|
||||
echo "Container $CONTAINER: $state"
|
||||
if curl -fsS -m 3 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
|
||||
echo "Receiver lokal erreichbar (127.0.0.1:8090/healthz ok)"
|
||||
else
|
||||
echo "WARNUNG: 127.0.0.1:8090/healthz nicht erreichbar"
|
||||
fi
|
||||
echo "Strg+C beendet. Rueckblick: $SINCE"
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
|
||||
# Faerbt und filtert; gibt bei jedem Treffer den Zaehler mit aus.
|
||||
colorize() {
|
||||
awk -v all="$ALL" '
|
||||
BEGIN { R="\033[31m"; G="\033[32m"; Y="\033[33m"; C="\033[36m"; B="\033[1m"; N="\033[0m"; posts=0 }
|
||||
{
|
||||
line = $0
|
||||
is_post = (line ~ /"POST /)
|
||||
is_gundi = (line ~ /\[GUNDI\]/)
|
||||
is_err = (line ~ /(Traceback|Error|Exception|Fehler|WARN|ALARM|STOERUNG)/) || (line ~ /" (4[0-9][0-9]|5[0-9][0-9]) /)
|
||||
if (!all && !is_post && !is_gundi && !is_err) next
|
||||
if (is_post) posts++
|
||||
col = ""
|
||||
if (is_err) col = R
|
||||
else if (is_gundi) col = G
|
||||
else if (is_post) col = C
|
||||
if (line ~ /" 30[18] /) col = Y # Redirect: Client folgt evtl. nicht mit POST
|
||||
printf "%s%s%s", col, line, N
|
||||
if (is_post) printf " %s(POST #%d)%s", B, posts, N
|
||||
printf "\n"
|
||||
fflush()
|
||||
}'
|
||||
}
|
||||
|
||||
pids=()
|
||||
cleanup() { kill "${pids[@]}" 2>/dev/null; exit 0; }
|
||||
trap cleanup INT TERM
|
||||
|
||||
docker logs -f -t --since "$SINCE" "$CONTAINER" 2>&1 \
|
||||
| sed -u 's/^/[app] /' | colorize &
|
||||
pids+=($!)
|
||||
|
||||
if [ -n "$NGINX_LOG" ] && [ -e "$NGINX_LOG" ]; then
|
||||
# Auf meles_dev3 gehoert das Log www-data:adm (0640) - ohne adm-Gruppe per sudo lesen.
|
||||
SUDO=""
|
||||
if [ ! -r "$NGINX_LOG" ]; then
|
||||
# sudo muss im Vordergrund nach dem Passwort fragen (in einer &-Pipeline haengt es);
|
||||
# danach liest "sudo -n" aus dem Credential-Cache.
|
||||
echo "nginx-Log nicht direkt lesbar - sudo-Passwort fuer das Mitlesen:"
|
||||
if sudo -v; then
|
||||
SUDO="sudo -n"
|
||||
else
|
||||
echo "Hinweis: kein sudo - nginx-Log wird nicht verfolgt" >&2
|
||||
NGINX_LOG=""
|
||||
fi
|
||||
fi
|
||||
if [ -n "$NGINX_LOG" ]; then
|
||||
$SUDO tail -n 300 -F "$NGINX_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[nginx] /' | colorize &
|
||||
pids+=($!)
|
||||
fi
|
||||
elif [ -n "$TRAEFIK_FILE" ]; then
|
||||
if [ -r "$TRAEFIK_FILE" ]; then
|
||||
tail -n 200 -F "$TRAEFIK_FILE" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[traefik] /' | colorize &
|
||||
pids+=($!)
|
||||
else
|
||||
echo "Hinweis: $TRAEFIK_FILE nicht lesbar - Traefik-Log wird nicht verfolgt" >&2
|
||||
fi
|
||||
elif [ -n "$TRAEFIK" ]; then
|
||||
if [ "$TRAEFIK" = "auto" ]; then
|
||||
TRAEFIK=$(docker ps --format '{{.Names}}' | grep -i traefik | head -n1)
|
||||
fi
|
||||
if [ -n "$TRAEFIK" ] && docker inspect "$TRAEFIK" >/dev/null 2>&1; then
|
||||
echo "Traefik-Container: $TRAEFIK (Access-Log: $TRAEFIK_INNER_LOG)"
|
||||
# Traefik schreibt den Access-Log in eine Datei im Container (--accesslog.filepath),
|
||||
# nicht nach stdout - daher docker exec tail. Fallback: docker logs.
|
||||
if docker exec "$TRAEFIK" test -r "$TRAEFIK_INNER_LOG" 2>/dev/null; then
|
||||
docker exec "$TRAEFIK" tail -n 300 -F "$TRAEFIK_INNER_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events|icu\.meles\.eu' | sed -u 's/^/[traefik] /' | colorize &
|
||||
else
|
||||
docker logs -f -t --since "$SINCE" "$TRAEFIK" 2>&1 | grep --line-buffered -E '/v2/events|icu\.meles\.eu|level=(error|fatal)' | sed -u 's/^/[traefik] /' | colorize &
|
||||
fi
|
||||
pids+=($!)
|
||||
else
|
||||
echo "Hinweis: kein Traefik-Container gefunden (-t NAME oder -f LOGFILE angeben)" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
wait
|
||||
Reference in New Issue
Block a user