Add environment configuration and enhance Shiny app with AWS integration
This commit is contained in:
parent
3873cb7b3f
commit
b35394c569
5
.env
Normal file
5
.env
Normal file
@ -0,0 +1,5 @@
|
||||
PG_DSN=postgresql://postgres:DFGk5s9H21lKao2K@136.243.41.58:7777/meles
|
||||
S3_BUCKET=trapper-meles
|
||||
S3_ACCESS_KEY=UG950FCGWFYRXSXXVUAH
|
||||
S3_SECRET_KEY=rXpspNhboZY6zNZi7djjq2QaXPA4uwsO9jXf4AXk
|
||||
S3_ENDPOINT=https://fsn1.your-objectstorage.com/
|
||||
@ -18,3 +18,5 @@ app:
|
||||
ocr_crop_w_frac: 0.42
|
||||
ocr_crop_h_frac: 0.22
|
||||
thumb_max: 512
|
||||
exiftool_timeout_seconds: 15
|
||||
job_sleep_seconds: 1.0
|
||||
|
||||
@ -5,9 +5,28 @@ services:
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CONFIG_YAML: /app/config.yaml
|
||||
DEBUG: "1"
|
||||
DEBUG: "0"
|
||||
ports:
|
||||
- "5678:5678"
|
||||
volumes:
|
||||
- ./:/app
|
||||
- ./config.yaml:/app/config.yaml:ro
|
||||
melesicu_shiny:
|
||||
build: ./shiny-app
|
||||
container_name: melesicu_shiny
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PG_DSN: ${PG_DSN}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
ENTRANCE_PREFIX: "icu/entrance/"
|
||||
PROCESSED_PREFIX: "icu/processed/"
|
||||
THUMB_PREFIX: "icu/thumbnails/"
|
||||
INVENTORY_TABLE: "cam_inventory"
|
||||
DEPLOYMENT_TABLE: "cam_deployments"
|
||||
AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY}
|
||||
AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY}
|
||||
AWS_DEFAULT_REGION: "us-east-1"
|
||||
AWS_S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
AWS_S3_ENDPOINT_URL: ${S3_ENDPOINT}
|
||||
ports:
|
||||
- "3838:3838"
|
||||
|
||||
115
main.py
115
main.py
@ -64,6 +64,8 @@ class AppConfig:
|
||||
ocr_crop_h_frac: float = 0.22 # bottom ~22% of height
|
||||
|
||||
thumb_max: int = 512
|
||||
exiftool_timeout_seconds: int = 15
|
||||
job_sleep_seconds: float = 1.0
|
||||
|
||||
|
||||
def load_config(path: str) -> AppConfig:
|
||||
@ -92,6 +94,8 @@ def load_config(path: str) -> AppConfig:
|
||||
ocr_crop_w_frac=float(app.get("ocr_crop_w_frac", 0.42)),
|
||||
ocr_crop_h_frac=float(app.get("ocr_crop_h_frac", 0.22)),
|
||||
thumb_max=int(app.get("thumb_max", 512)),
|
||||
exiftool_timeout_seconds=int(app.get("exiftool_timeout_seconds", 15)),
|
||||
job_sleep_seconds=float(app.get("job_sleep_seconds", 1.0)),
|
||||
)
|
||||
|
||||
|
||||
@ -210,7 +214,7 @@ def _parse_ocr_datetime(*texts: str) -> datetime:
|
||||
raise ValueError(f"OCR timestamp not found. OCR1='{texts[0] if texts else ''}' OCR2='{texts[1] if len(texts) > 1 else ''}'")
|
||||
|
||||
|
||||
def exif_write_with_exiftool(jpg_in: bytes, dt_original_local: datetime, dt_digitized_utc: datetime) -> bytes:
|
||||
def exif_write_with_exiftool(jpg_in: bytes, dt_original_local: datetime, dt_digitized_utc: datetime, *, timeout_seconds: int) -> bytes:
|
||||
"""
|
||||
DateTimeOriginal = OCR time (camera local) + OffsetTimeOriginal
|
||||
DateTimeDigitized = servertime (UTC) + OffsetTimeDigitized=+00:00
|
||||
@ -237,7 +241,7 @@ def exif_write_with_exiftool(jpg_in: bytes, dt_original_local: datetime, dt_digi
|
||||
f"-OffsetTime={otd}",
|
||||
in_path,
|
||||
]
|
||||
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_seconds)
|
||||
|
||||
with open(in_path, "rb") as f:
|
||||
return f.read()
|
||||
@ -320,6 +324,41 @@ def job_set_status(cur, uuid: str, status: str, *, err: Optional[str] = None, pr
|
||||
""", (status, status, err, processed_key, thumb_key, uuid))
|
||||
|
||||
|
||||
def job_get(cur, uuid: str):
|
||||
cur.execute("""
|
||||
SELECT status, processed_jpg_key, thumbnail_key, has_categories
|
||||
FROM remote_cam.import_job
|
||||
WHERE image_uuid = %s;
|
||||
""", (uuid,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"status": row[0],
|
||||
"processed_jpg_key": row[1],
|
||||
"thumbnail_key": row[2],
|
||||
"has_categories": row[3],
|
||||
}
|
||||
|
||||
|
||||
STATUS_ORDER = [
|
||||
"NEW",
|
||||
"META_OK",
|
||||
"OCR_OK",
|
||||
"EXIF_OK",
|
||||
"MOVED",
|
||||
"THUMB_OK",
|
||||
"CATEGORIES_OK",
|
||||
"CLEANED",
|
||||
]
|
||||
|
||||
|
||||
def status_rank(status: Optional[str]) -> int:
|
||||
if status in STATUS_ORDER:
|
||||
return STATUS_ORDER.index(status)
|
||||
return -1
|
||||
|
||||
|
||||
def insert_resource(cur, uuid: str, typ: str):
|
||||
cur.execute("""
|
||||
INSERT INTO remote_cam.resource(resource_uuid, type, import_ts)
|
||||
@ -424,6 +463,16 @@ def update_ocr_ts(cur, uuid: str, ocr_ts: datetime):
|
||||
""", (ocr_ts, uuid))
|
||||
|
||||
|
||||
def get_ocr_ts(cur, uuid: str) -> Optional[datetime]:
|
||||
cur.execute("""
|
||||
SELECT ocr_ts
|
||||
FROM remote_cam.metadata
|
||||
WHERE metadata_uuid = %s;
|
||||
""", (uuid,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def insert_categories(cur, uuid: str, cat_json: dict):
|
||||
version = cat_json.get("version")
|
||||
cats = cat_json.get("categories") or []
|
||||
@ -528,54 +577,97 @@ def collect_ready_sets(s3: S3, cfg: AppConfig) -> list[EntranceSet]:
|
||||
|
||||
def process_one(cur, s3: S3, cfg: AppConfig, s: EntranceSet):
|
||||
job_upsert_new(cur, s.uuid, s.jpg_key, s.meta_key, s.cat_key)
|
||||
job = job_get(cur, s.uuid) or {}
|
||||
status_now = job.get("status", "NEW")
|
||||
|
||||
# 1) metadata.json -> DB
|
||||
meta = json.loads(s3.get_bytes(s.meta_key))
|
||||
insert_metadata(cur, s.uuid, meta)
|
||||
insert_resource(cur, s.uuid, "metadata")
|
||||
if status_rank("META_OK") >= status_rank(status_now):
|
||||
job_set_status(cur, s.uuid, "META_OK")
|
||||
status_now = "META_OK"
|
||||
|
||||
# 2) JPG -> OCR bottom-right -> ocr_ts in CustomerTimezone
|
||||
jpg_bytes = s3.get_bytes(s.jpg_key)
|
||||
ocr_ts = None
|
||||
if status_rank(status_now) < status_rank("OCR_OK"):
|
||||
try:
|
||||
customer_tzinfo, _ = parse_customer_tz(meta)
|
||||
ocr_ts = ocr_extract_timestamp(jpg_bytes, cfg.ocr_crop_w_frac, cfg.ocr_crop_h_frac, customer_tzinfo)
|
||||
update_ocr_ts(cur, s.uuid, ocr_ts)
|
||||
insert_resource(cur, s.uuid, "image")
|
||||
job_set_status(cur, s.uuid, "OCR_OK")
|
||||
status_now = "OCR_OK"
|
||||
except Exception as e:
|
||||
job_set_status(cur, s.uuid, "OCR_FAIL", err=str(e))
|
||||
status_now = "OCR_FAIL"
|
||||
elif status_rank(status_now) >= status_rank("OCR_OK"):
|
||||
ocr_ts = get_ocr_ts(cur, s.uuid)
|
||||
|
||||
# 3) EXIF write:
|
||||
# DateTimeOriginal = ocr_ts (local, keep offset tags)
|
||||
# DateTimeDigitized = servertime (UTC)
|
||||
jpg_exif = jpg_bytes
|
||||
if status_rank(status_now) < status_rank("MOVED"):
|
||||
if ocr_ts is None:
|
||||
job_set_status(cur, s.uuid, "EXIF_SKIPPED", err="OCR missing")
|
||||
status_now = "EXIF_SKIPPED"
|
||||
else:
|
||||
try:
|
||||
servertime = parse_servertime(meta)
|
||||
jpg_exif = exif_write_with_exiftool(jpg_bytes, ocr_ts, servertime)
|
||||
jpg_exif = exif_write_with_exiftool(
|
||||
jpg_bytes,
|
||||
ocr_ts,
|
||||
servertime,
|
||||
timeout_seconds=cfg.exiftool_timeout_seconds,
|
||||
)
|
||||
job_set_status(cur, s.uuid, "EXIF_OK")
|
||||
status_now = "EXIF_OK"
|
||||
except Exception as e:
|
||||
job_set_status(cur, s.uuid, "EXIF_FAIL", err=str(e))
|
||||
status_now = "EXIF_FAIL"
|
||||
|
||||
# 4) Upload processed JPG (EXIF patched)
|
||||
processed_key = f"{cfg.processed_prefix}{s.uuid}.jpg"
|
||||
processed_key = job.get("processed_jpg_key") or f"{cfg.processed_prefix}{s.uuid}.jpg"
|
||||
if status_rank(status_now) < status_rank("MOVED"):
|
||||
s3.put_bytes(processed_key, jpg_exif, "image/jpeg")
|
||||
s3.head(processed_key)
|
||||
job_set_status(cur, s.uuid, "MOVED", processed_key=processed_key)
|
||||
status_now = "MOVED"
|
||||
|
||||
# 5) Thumbnail
|
||||
thumb_key = job.get("thumbnail_key") or f"{cfg.thumb_prefix}{s.uuid}.jpg"
|
||||
if status_rank(status_now) < status_rank("THUMB_OK"):
|
||||
if status_rank(status_now) >= status_rank("MOVED") and jpg_exif is jpg_bytes:
|
||||
jpg_exif = s3.get_bytes(processed_key)
|
||||
thumb_bytes = make_thumbnail_bytes(jpg_exif, cfg.thumb_max)
|
||||
thumb_key = f"{cfg.thumb_prefix}{s.uuid}.jpg"
|
||||
s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg")
|
||||
s3.head(thumb_key)
|
||||
job_set_status(cur, s.uuid, "THUMB_OK", thumb_key=thumb_key)
|
||||
status_now = "THUMB_OK"
|
||||
|
||||
# 6) Optional categories
|
||||
if s.cat_key:
|
||||
if s.cat_key and status_rank(status_now) < status_rank("CATEGORIES_OK"):
|
||||
cat_json = json.loads(s3.get_bytes(s.cat_key))
|
||||
insert_categories(cur, s.uuid, cat_json)
|
||||
insert_resource(cur, s.uuid, "classification")
|
||||
job_set_status(cur, s.uuid, "CATEGORIES_OK")
|
||||
status_now = "CATEGORIES_OK"
|
||||
|
||||
# 7) Cleanup entrance only after verify
|
||||
if status_rank(status_now) >= status_rank("THUMB_OK"):
|
||||
try:
|
||||
s3.head(processed_key)
|
||||
s3.head(thumb_key)
|
||||
to_delete = [s.jpg_key, s.meta_key]
|
||||
if s.cat_key:
|
||||
to_delete.append(s.cat_key)
|
||||
s3.delete_keys(to_delete)
|
||||
job_set_status(cur, s.uuid, "CLEANED")
|
||||
status_now = "CLEANED"
|
||||
except Exception as e:
|
||||
job_set_status(cur, s.uuid, "CLEANUP_SKIPPED", err=str(e))
|
||||
|
||||
|
||||
def run_once(cfg: AppConfig) -> int:
|
||||
@ -585,10 +677,6 @@ def run_once(cfg: AppConfig) -> int:
|
||||
return 0
|
||||
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
for s in ready:
|
||||
try:
|
||||
with db.transaction():
|
||||
@ -600,6 +688,8 @@ def run_once(cfg: AppConfig) -> int:
|
||||
job_upsert_new(cur, s.uuid, s.jpg_key, s.meta_key, s.cat_key)
|
||||
job_set_status(cur, s.uuid, "ERROR", err=str(e))
|
||||
print(f"[ERROR] {s.uuid}: {e}")
|
||||
if cfg.job_sleep_seconds > 0:
|
||||
time.sleep(cfg.job_sleep_seconds)
|
||||
return len(ready)
|
||||
|
||||
|
||||
@ -607,6 +697,11 @@ def main():
|
||||
cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml")
|
||||
cfg = load_config(cfg_path)
|
||||
|
||||
with psycopg.connect(cfg.pg_dsn) as db:
|
||||
with db.cursor() as cur:
|
||||
db_init(cur)
|
||||
db.commit()
|
||||
|
||||
while True:
|
||||
n = run_once(cfg)
|
||||
if n == 0:
|
||||
|
||||
22
shiny-app/Dockerfile
Normal file
22
shiny-app/Dockerfile
Normal file
@ -0,0 +1,22 @@
|
||||
FROM rocker/shiny:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq-dev \
|
||||
libcurl4-openssl-dev \
|
||||
libssl-dev \
|
||||
libxml2-dev \
|
||||
curl \
|
||||
unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
|
||||
&& unzip awscliv2.zip \
|
||||
&& ./aws/install \
|
||||
&& rm -rf awscliv2.zip aws
|
||||
|
||||
RUN R -e "install.packages(c('DBI','RPostgres','dplyr','DT','glue','shiny'), repos='https://cloud.r-project.org')"
|
||||
|
||||
WORKDIR /srv/shiny-server
|
||||
COPY app.R /srv/shiny-server/app.R
|
||||
|
||||
EXPOSE 3838
|
||||
517
shiny-app/app.R
Normal file
517
shiny-app/app.R
Normal file
@ -0,0 +1,517 @@
|
||||
library(shiny)
|
||||
library(DBI)
|
||||
library(RPostgres)
|
||||
library(dplyr)
|
||||
library(DT)
|
||||
library(glue)
|
||||
|
||||
pg_dsn <- Sys.getenv("PG_DSN", "")
|
||||
s3_bucket <- Sys.getenv("S3_BUCKET", "")
|
||||
s3_endpoint <- Sys.getenv("S3_ENDPOINT", Sys.getenv("AWS_S3_ENDPOINT", Sys.getenv("AWS_S3_ENDPOINT_URL", "")))
|
||||
aws_access_key <- Sys.getenv("S3_ACCESS_KEY", Sys.getenv("AWS_ACCESS_KEY_ID", ""))
|
||||
aws_secret_key <- Sys.getenv("S3_SECRET_KEY", Sys.getenv("AWS_SECRET_ACCESS_KEY", ""))
|
||||
aws_cli <- Sys.getenv("AWS_CLI", "aws")
|
||||
thumb_prefix <- Sys.getenv("THUMB_PREFIX", "icu/thumbnails/")
|
||||
processed_prefix <- Sys.getenv("PROCESSED_PREFIX", "icu/processed/")
|
||||
entrance_prefix <- Sys.getenv("ENTRANCE_PREFIX", "icu/entrance/")
|
||||
processed_prefix <- Sys.getenv("PROCESSED_PREFIX", "icu/processed/")
|
||||
thumb_prefix <- Sys.getenv("THUMB_PREFIX", "icu/thumbnails/")
|
||||
|
||||
inventory_table <- Sys.getenv("INVENTORY_TABLE", "")
|
||||
deployment_table <- Sys.getenv("DEPLOYMENT_TABLE", "")
|
||||
|
||||
if (pg_dsn == "") {
|
||||
stop("PG_DSN env var is required")
|
||||
}
|
||||
if (s3_bucket == "") {
|
||||
stop("S3_BUCKET env var is required")
|
||||
}
|
||||
if (s3_endpoint == "") {
|
||||
stop("S3_ENDPOINT/AWS_S3_ENDPOINT env var is required")
|
||||
}
|
||||
if (aws_access_key == "" || aws_secret_key == "") {
|
||||
stop("S3_ACCESS_KEY/S3_SECRET_KEY or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY required")
|
||||
}
|
||||
|
||||
Sys.setenv(
|
||||
AWS_ACCESS_KEY_ID = aws_access_key,
|
||||
AWS_SECRET_ACCESS_KEY = aws_secret_key,
|
||||
AWS_EC2_METADATA_DISABLED = "true"
|
||||
)
|
||||
|
||||
parse_pg_dsn <- function(dsn) {
|
||||
if (grepl("^postgresql://", dsn)) {
|
||||
m <- regexec("^postgresql://([^:]+):([^@]+)@([^:/]+):?(\\d+)?/(.+)$", dsn)
|
||||
r <- regmatches(dsn, m)[[1]]
|
||||
if (length(r) == 0) stop("Invalid PG_DSN URL format")
|
||||
return(list(
|
||||
user = r[2],
|
||||
password = r[3],
|
||||
host = r[4],
|
||||
port = ifelse(r[5] == "", 5432, as.integer(r[5])),
|
||||
dbname = r[6]
|
||||
))
|
||||
}
|
||||
|
||||
if (grepl("host=", dsn)) {
|
||||
parts <- strsplit(dsn, "\\s+")[[1]]
|
||||
kv <- lapply(parts, function(p) strsplit(p, "=", fixed = TRUE)[[1]])
|
||||
vals <- setNames(lapply(kv, `[`, 2), lapply(kv, `[`, 1))
|
||||
return(list(
|
||||
user = vals$user,
|
||||
password = vals$password,
|
||||
host = vals$host,
|
||||
port = as.integer(vals$port),
|
||||
dbname = vals$dbname
|
||||
))
|
||||
}
|
||||
|
||||
return(list(dbname = dsn))
|
||||
}
|
||||
|
||||
pg <- parse_pg_dsn(pg_dsn)
|
||||
pg_con <- do.call(dbConnect, c(list(RPostgres::Postgres()), pg))
|
||||
onStop(function() dbDisconnect(pg_con))
|
||||
|
||||
table_exists <- function(con, table_name) {
|
||||
res <- dbGetQuery(con, "SELECT to_regclass($1) AS t", params = list(table_name))
|
||||
!is.na(res$t[1])
|
||||
}
|
||||
|
||||
fetch_cameras <- function() {
|
||||
if (inventory_table != "" && table_exists(pg_con, inventory_table)) {
|
||||
q <- glue("SELECT DISTINCT cam_id FROM {inventory_table} ORDER BY 1")
|
||||
return(dbGetQuery(pg_con, q)$cam_id)
|
||||
}
|
||||
character(0)
|
||||
}
|
||||
|
||||
fetch_projects <- function() {
|
||||
if (deployment_table != "" && table_exists(pg_con, deployment_table)) {
|
||||
q <- glue("SELECT DISTINCT project FROM {deployment_table} ORDER BY 1")
|
||||
return(dbGetQuery(pg_con, q)$project)
|
||||
}
|
||||
character(0)
|
||||
}
|
||||
|
||||
build_query <- function(from_date, to_date, camera_id, project_name, limit_rows, offset_rows, use_date_filter) {
|
||||
base <- "
|
||||
SELECT
|
||||
m.metadata_uuid,
|
||||
m.import_ts,
|
||||
m.ocr_ts,
|
||||
m.\"CustomerTimezone\",
|
||||
m.\"SignalStrength\",
|
||||
m.\"Temperature\",
|
||||
m.imei,
|
||||
m.servertime,
|
||||
ci.cam_id,
|
||||
d.project,
|
||||
c.category,
|
||||
c.score
|
||||
FROM remote_cam.metadata m
|
||||
LEFT JOIN cam_inventory ci ON ci.imei = m.imei
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT d.project
|
||||
FROM cam_deployments d
|
||||
WHERE d.camera_id = ci.cam_id
|
||||
AND m.servertime >= d.deployment_start
|
||||
AND (d.deployment_end IS NULL OR m.servertime < d.deployment_end)
|
||||
ORDER BY d.deployment_start DESC NULLS LAST
|
||||
LIMIT 1
|
||||
) d ON TRUE
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT category, score
|
||||
FROM remote_cam.category c
|
||||
WHERE c.image_uuid = m.metadata_uuid
|
||||
ORDER BY score DESC NULLS LAST
|
||||
LIMIT 1
|
||||
) c ON TRUE
|
||||
"
|
||||
|
||||
params <- list()
|
||||
|
||||
if (camera_id != "" && inventory_table != "" && table_exists(pg_con, inventory_table)) {
|
||||
base <- paste0(base, " WHERE ci.cam_id = $", length(params) + 1)
|
||||
params <- append(params, list(camera_id))
|
||||
}
|
||||
|
||||
if (project_name != "" && deployment_table != "" && table_exists(pg_con, deployment_table)) {
|
||||
prefix <- if (length(params) == 0) " WHERE " else " AND "
|
||||
base <- paste0(base, prefix, " m.imei IN (SELECT ci2.imei FROM cam_inventory ci2 JOIN ",
|
||||
deployment_table, " d ON d.camera_id = ci2.cam_id WHERE d.project = $",
|
||||
length(params) + 1, ")")
|
||||
params <- append(params, list(project_name))
|
||||
}
|
||||
|
||||
if (camera_id == "" && project_name == "" && !use_date_filter) {
|
||||
base <- paste0(base, " ORDER BY m.import_ts DESC LIMIT $", length(params) + 1, " OFFSET $", length(params) + 2)
|
||||
params <- append(params, list(as.integer(limit_rows), as.integer(offset_rows)))
|
||||
return(list(sql = base, params = params))
|
||||
}
|
||||
|
||||
if (use_date_filter) {
|
||||
prefix <- if (length(params) == 0) " WHERE " else " AND "
|
||||
base <- paste0(base, prefix, " m.import_ts >= $", length(params) + 1, " AND m.import_ts < $", length(params) + 2)
|
||||
params <- append(params, list(from_date, to_date))
|
||||
}
|
||||
|
||||
base <- paste0(base, " ORDER BY m.import_ts DESC LIMIT $", length(params) + 1, " OFFSET $", length(params) + 2)
|
||||
params <- append(params, list(as.integer(limit_rows), as.integer(offset_rows)))
|
||||
list(sql = base, params = params)
|
||||
}
|
||||
|
||||
normalize_endpoint <- function(url) {
|
||||
if (url == "") return("")
|
||||
u <- url
|
||||
if (!grepl("^https?://", u)) {
|
||||
u <- paste0("https://", u)
|
||||
}
|
||||
sub("/+$", "", u)
|
||||
}
|
||||
|
||||
s3_endpoint <- normalize_endpoint(s3_endpoint)
|
||||
|
||||
if (aws_access_key == "" || aws_secret_key == "" || s3_endpoint == "") {
|
||||
stop("S3_ACCESS_KEY/S3_SECRET_KEY or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY and S3_ENDPOINT are required")
|
||||
}
|
||||
|
||||
normalize_endpoint <- function(url) {
|
||||
if (url == "") return("")
|
||||
u <- url
|
||||
if (!grepl("^https?://", u)) {
|
||||
u <- paste0("https://", u)
|
||||
}
|
||||
sub("/+$", "", u)
|
||||
}
|
||||
|
||||
s3_endpoint <- normalize_endpoint(s3_endpoint)
|
||||
cache_dir <- Sys.getenv("THUMB_CACHE_DIR", "/tmp/thumb_cache")
|
||||
dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
addResourcePath("thumbs", cache_dir)
|
||||
|
||||
presigned_url <- function(key, expires = 600) {
|
||||
if (is.null(key) || is.na(key) || key == "") return(NULL)
|
||||
key <- sub(paste0("^", s3_bucket, "/"), "", key)
|
||||
cmd <- sprintf(
|
||||
"\"%s\" --endpoint-url %s s3 presign s3://%s/%s --expires-in %d",
|
||||
aws_cli, s3_endpoint, s3_bucket, key, as.integer(expires)
|
||||
)
|
||||
tryCatch({
|
||||
out <- system(cmd, intern = TRUE)
|
||||
if (length(out) == 0) return(NULL)
|
||||
out[1]
|
||||
}, error = function(e) {
|
||||
message(sprintf("[DEBUG] presign error: %s", e$message))
|
||||
NULL
|
||||
})
|
||||
}
|
||||
|
||||
cache_thumbnail <- function(uuid) {
|
||||
if (is.null(uuid) || is.na(uuid)) return(NULL)
|
||||
local_path <- file.path(cache_dir, paste0(uuid, ".jpg"))
|
||||
if (file.exists(local_path)) return(local_path)
|
||||
key <- paste0(thumb_prefix, uuid, ".jpg")
|
||||
cmd <- sprintf(
|
||||
"\"%s\" --endpoint-url %s s3 cp s3://%s/%s \"%s\" --only-show-errors",
|
||||
aws_cli, s3_endpoint, s3_bucket, key, local_path
|
||||
)
|
||||
tryCatch({
|
||||
system(cmd, ignore.stdout = TRUE, ignore.stderr = TRUE)
|
||||
if (file.exists(local_path)) return(local_path)
|
||||
NULL
|
||||
}, error = function(e) {
|
||||
message(sprintf("[DEBUG] cache thumb error: %s", e$message))
|
||||
NULL
|
||||
})
|
||||
}
|
||||
|
||||
ui <- fluidPage(
|
||||
titlePanel("MelesICU - Image Browser"),
|
||||
tags$script(HTML("
|
||||
function melesThumbClick(el){
|
||||
var uuid = el.getAttribute('data-uuid');
|
||||
if(uuid){
|
||||
Shiny.setInputValue('thumb_click', uuid, {priority: 'event'});
|
||||
}
|
||||
}
|
||||
var melesScrollLock = false;
|
||||
window.addEventListener('scroll', function() {
|
||||
if (melesScrollLock) return;
|
||||
var nearBottom = (window.innerHeight + window.scrollY) >= (document.body.offsetHeight - 200);
|
||||
var canLoad = Shiny && Shiny.shinyapp && Shiny.shinyapp.$outputValues && Shiny.shinyapp.$outputValues.has_more_flag;
|
||||
var isLoading = Shiny && Shiny.shinyapp && Shiny.shinyapp.$outputValues && Shiny.shinyapp.$outputValues.loading_flag;
|
||||
if (nearBottom && canLoad && !isLoading) {
|
||||
melesScrollLock = true;
|
||||
Shiny.setInputValue('scroll_load', Date.now(), {priority: 'event'});
|
||||
setTimeout(function(){ melesScrollLock = false; }, 1000);
|
||||
}
|
||||
});
|
||||
")),
|
||||
sidebarLayout(
|
||||
sidebarPanel(
|
||||
dateRangeInput("date_range", "Datum", start = Sys.Date() - 7, end = Sys.Date()),
|
||||
checkboxInput("use_date_filter", "Datumsfilter aktiv", value = TRUE),
|
||||
selectInput("camera_id", "Kamera (cam_id)", choices = c("ALL", fetch_cameras()), selected = "ALL"),
|
||||
selectInput("project_name", "Projekt", choices = c("ALL", fetch_projects()), selected = "ALL"),
|
||||
numericInput("limit_rows", "Eintraege pro Seite", value = 20, min = 10, max = 2000, step = 10),
|
||||
numericInput("page", "Seite", value = 1, min = 1, step = 1),
|
||||
fluidRow(
|
||||
column(6, actionButton("prev_page", "Zurueck")),
|
||||
column(6, actionButton("next_page", "Vor"))
|
||||
),
|
||||
actionButton("apply_filters", "Filter anwenden")
|
||||
),
|
||||
mainPanel(
|
||||
tags$style(HTML("
|
||||
.spinner {
|
||||
margin: 20px 0;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border: 4px solid #ddd;
|
||||
border-top-color: #333;
|
||||
border-radius: 50%;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
")),
|
||||
conditionalPanel("output.loading_flag", tags$div(class = "spinner")),
|
||||
uiOutput("debug_info"),
|
||||
uiOutput("thumb_grid")
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
server <- function(input, output, session) {
|
||||
rv <- reactiveValues(data = NULL, selected = NULL, loading = FALSE, has_more = TRUE, debug = NULL)
|
||||
cache_env <- new.env(parent = emptyenv())
|
||||
cache_keys <- character(0)
|
||||
url_cache_env <- new.env(parent = emptyenv())
|
||||
url_cache_keys <- character(0)
|
||||
|
||||
load_data <- function() {
|
||||
from_date <- as.POSIXct(input$date_range[1], tz = "UTC")
|
||||
to_date <- as.POSIXct(input$date_range[2] + 1, tz = "UTC")
|
||||
cam <- ifelse(input$camera_id == "ALL", "", input$camera_id)
|
||||
proj <- ifelse(input$project_name == "ALL", "", input$project_name)
|
||||
page <- max(1, as.integer(input$page))
|
||||
limit <- as.integer(input$limit_rows)
|
||||
offset <- (page - 1) * limit
|
||||
|
||||
key <- paste(from_date, to_date, cam, proj, input$use_date_filter, limit, offset, sep = "|")
|
||||
if (exists(key, envir = cache_env, inherits = FALSE)) {
|
||||
rv$data <- get(key, envir = cache_env, inherits = FALSE)
|
||||
return(invisible(NULL))
|
||||
}
|
||||
|
||||
q <- build_query(from_date, to_date, cam, proj, limit, offset, input$use_date_filter)
|
||||
rv$loading <- TRUE
|
||||
on.exit({ rv$loading <- FALSE }, add = TRUE)
|
||||
data <- dbGetQuery(pg_con, q$sql, params = q$params)
|
||||
assign(key, data, envir = cache_env)
|
||||
cache_keys <<- c(cache_keys, key)
|
||||
if (length(cache_keys) > 20) {
|
||||
drop <- head(cache_keys, length(cache_keys) - 20)
|
||||
for (k in drop) rm(list = k, envir = cache_env)
|
||||
cache_keys <<- tail(cache_keys, 20)
|
||||
}
|
||||
rv$data <- data
|
||||
rv$has_more <- nrow(data) >= limit
|
||||
|
||||
# Batch prefetch first 10 thumbnails in parallel
|
||||
if (!is.null(data) && nrow(data) > 0) {
|
||||
uuids <- head(data$metadata_uuid, 10)
|
||||
tryCatch({
|
||||
parallel::mclapply(uuids, cache_thumbnail, mc.cores = 4)
|
||||
}, error = function(e) {
|
||||
message(sprintf("[DEBUG] prefetch error: %s", e$message))
|
||||
})
|
||||
}
|
||||
|
||||
if (!is.null(data) && nrow(data) > 0) {
|
||||
sample_uuid <- data$metadata_uuid[1]
|
||||
thumb_key <- paste0(thumb_prefix, sample_uuid, ".jpg")
|
||||
url <- presigned_url(thumb_key)
|
||||
rv$debug <- list(
|
||||
count = nrow(data),
|
||||
sample_uuid = sample_uuid,
|
||||
thumb_key = thumb_key,
|
||||
url = url
|
||||
)
|
||||
message(sprintf("[DEBUG] count=%d sample_uuid=%s thumb_key=%s url=%s",
|
||||
nrow(data), sample_uuid, thumb_key, ifelse(is.null(url), "NULL", url)))
|
||||
} else {
|
||||
rv$debug <- list(count = 0)
|
||||
message("[DEBUG] count=0")
|
||||
}
|
||||
}
|
||||
|
||||
observeEvent(input$apply_filters, {
|
||||
if (rv$loading) {
|
||||
showNotification("Noch am Laden – bitte warten.", type = "message")
|
||||
return(NULL)
|
||||
}
|
||||
load_data()
|
||||
}, ignoreInit = TRUE)
|
||||
|
||||
observeEvent(input$prev_page, {
|
||||
if (rv$loading) return(NULL)
|
||||
new_page <- max(1, as.integer(input$page) - 1)
|
||||
updateNumericInput(session, "page", value = new_page)
|
||||
load_data()
|
||||
}, ignoreInit = TRUE)
|
||||
|
||||
observeEvent(input$next_page, {
|
||||
if (rv$loading) return(NULL)
|
||||
new_page <- as.integer(input$page) + 1
|
||||
updateNumericInput(session, "page", value = new_page)
|
||||
load_data()
|
||||
}, ignoreInit = TRUE)
|
||||
|
||||
observeEvent(input$scroll_load, {
|
||||
if (rv$loading || !rv$has_more) return(NULL)
|
||||
new_page <- as.integer(input$page) + 1
|
||||
updateNumericInput(session, "page", value = new_page)
|
||||
load_data()
|
||||
}, ignoreInit = TRUE)
|
||||
|
||||
observeEvent(TRUE, {
|
||||
if (rv$loading) return(NULL)
|
||||
load_data()
|
||||
}, once = TRUE)
|
||||
|
||||
output$loading_flag <- reactive({ rv$loading })
|
||||
outputOptions(output, "loading_flag", suspendWhenHidden = FALSE)
|
||||
|
||||
output$has_more_flag <- reactive({ rv$has_more })
|
||||
outputOptions(output, "has_more_flag", suspendWhenHidden = FALSE)
|
||||
|
||||
format_cet <- function(ts) {
|
||||
if (is.null(ts) || is.na(ts)) return("-")
|
||||
format(as.POSIXct(ts, tz = "CET"), "%Y-%m-%d %H:%M:%S %Z")
|
||||
}
|
||||
|
||||
format_ts <- function(ts) {
|
||||
if (is.null(ts) || is.na(ts)) return("-")
|
||||
format(as.POSIXct(ts, tz = "CET"), "%Y-%m-%d %H:%M:%S %Z")
|
||||
}
|
||||
|
||||
format_diff <- function(server_ts, camera_ts) {
|
||||
if (is.null(server_ts) || is.na(server_ts) || is.null(camera_ts) || is.na(camera_ts)) return("-")
|
||||
secs <- as.numeric(difftime(server_ts, camera_ts, units = "secs"))
|
||||
sign <- ifelse(secs >= 0, "+", "-")
|
||||
secs <- abs(secs)
|
||||
hh <- floor(secs / 3600)
|
||||
mm <- floor((secs %% 3600) / 60)
|
||||
ss <- floor(secs %% 60)
|
||||
sprintf("%s%02d:%02d:%02d", sign, hh, mm, ss)
|
||||
}
|
||||
|
||||
output$thumb_grid <- renderUI({
|
||||
dat <- rv$data
|
||||
if (is.null(dat) || nrow(dat) == 0) {
|
||||
return(tags$div("Keine Ergebnisse."))
|
||||
}
|
||||
|
||||
cards <- lapply(seq_len(nrow(dat)), function(i) {
|
||||
row <- dat[i, ]
|
||||
local_path <- cache_thumbnail(row$metadata_uuid)
|
||||
if (is.null(local_path)) return(NULL)
|
||||
url <- paste0("thumbs/", basename(local_path))
|
||||
tags$div(
|
||||
style = "display:inline-block; width: 220px; margin: 6px; vertical-align: top;",
|
||||
tags$div(class = "ph", style = "width: 220px; height: 220px; background:#eee; display:block;"),
|
||||
tags$img(
|
||||
src = url,
|
||||
style = "width: 220px; height: 220px; object-fit: cover; display:block; cursor: pointer;",
|
||||
loading = "lazy",
|
||||
onload = "this.parentNode.querySelector('.ph').style.display='none';",
|
||||
`data-uuid` = row$metadata_uuid,
|
||||
onclick = "melesThumbClick(this)"
|
||||
),
|
||||
tags$div(style = "font-size: 12px; color: #222;",
|
||||
glue("CamID: {row$cam_id}")),
|
||||
tags$div(style = "font-size: 12px; color: #444;",
|
||||
glue("Serverzeit: {format_cet(row$servertime)}")),
|
||||
tags$div(style = "font-size: 12px; color: #444;",
|
||||
glue("Kamerazeit: {format_ts(row$ocr_ts)}")),
|
||||
tags$div(style = "font-size: 12px; color: #444;",
|
||||
glue("AI classification: {row$category} ({row$score})")),
|
||||
{
|
||||
delta <- format_diff(row$servertime, row$ocr_ts)
|
||||
delta_secs <- if (delta == "-") NA else as.numeric(difftime(row$servertime, row$ocr_ts, units = "secs"))
|
||||
is_warn <- !is.na(delta_secs) && abs(delta_secs) > 600
|
||||
style <- if (is_warn) "font-size: 12px; color: #c00; font-weight: bold;" else "font-size: 12px; color: #444;"
|
||||
tags$div(style = style, glue("Delta: {delta}"))
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
tags$div(cards)
|
||||
})
|
||||
|
||||
output$debug_info <- renderUI({
|
||||
if (is.null(rv$debug)) return(NULL)
|
||||
cnt <- rv$debug$count
|
||||
if (is.null(cnt)) return(NULL)
|
||||
if (cnt == 0) {
|
||||
return(tags$div(style = "font-size:12px; color:#a00; margin-bottom:8px;",
|
||||
"Debug: 0 Treffer."))
|
||||
}
|
||||
tags$div(style = "font-size:12px; color:#555; margin-bottom:8px;",
|
||||
glue("Debug: Treffer={cnt}, Sample UUID={rv$debug$sample_uuid}"),
|
||||
tags$br(),
|
||||
glue("Thumb-Key: {rv$debug$thumb_key}"),
|
||||
tags$br(),
|
||||
if (!is.null(rv$debug$url)) tags$a(href = rv$debug$url, "Sample Thumbnail (Link)", target = "_blank") else "Sample URL: NULL")
|
||||
})
|
||||
|
||||
observeEvent(input$thumb_click, {
|
||||
uuid <- input$thumb_click
|
||||
dat <- rv$data
|
||||
if (is.null(dat)) return(NULL)
|
||||
row <- dat[dat$metadata_uuid == uuid, ]
|
||||
if (nrow(row) != 1) return(NULL)
|
||||
full_key <- paste0(processed_prefix, row$metadata_uuid, ".jpg")
|
||||
presigned_cached <- function(key, expires = 600) {
|
||||
now <- Sys.time()
|
||||
if (exists(key, envir = url_cache_env, inherits = FALSE)) {
|
||||
entry <- get(key, envir = url_cache_env, inherits = FALSE)
|
||||
if (is.list(entry) && !is.null(entry$expires_at) && now < entry$expires_at) {
|
||||
return(entry$url)
|
||||
}
|
||||
}
|
||||
u <- presigned_url(key, expires = expires)
|
||||
if (!is.null(u)) {
|
||||
assign(key, list(url = u, expires_at = now + expires - 30), envir = url_cache_env)
|
||||
url_cache_keys <<- c(url_cache_keys, key)
|
||||
if (length(url_cache_keys) > 200) {
|
||||
drop <- head(url_cache_keys, length(url_cache_keys) - 200)
|
||||
for (k in drop) rm(list = k, envir = url_cache_env)
|
||||
url_cache_keys <<- tail(url_cache_keys, 200)
|
||||
}
|
||||
}
|
||||
u
|
||||
}
|
||||
|
||||
url <- presigned_cached(full_key)
|
||||
if (is.null(url)) {
|
||||
showModal(modalDialog(
|
||||
title = glue("Bild {uuid}"),
|
||||
"Full image nicht gefunden.",
|
||||
easyClose = TRUE,
|
||||
footer = NULL
|
||||
))
|
||||
return(NULL)
|
||||
}
|
||||
showModal(modalDialog(
|
||||
title = glue("Bild {uuid}"),
|
||||
tags$img(src = url, style = "max-width: 100%; height: auto;"),
|
||||
easyClose = TRUE,
|
||||
footer = modalButton("Schließen"),
|
||||
size = "l"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
shinyApp(ui, server)
|
||||
Loading…
x
Reference in New Issue
Block a user