diff --git a/.env b/.env new file mode 100644 index 0000000..cee2f7b --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +PG_DSN=postgresql://postgres:DFGk5s9H21lKao2K@136.243.41.58:7777/meles +S3_BUCKET=trapper-meles +S3_ACCESS_KEY=UG950FCGWFYRXSXXVUAH +S3_SECRET_KEY=rXpspNhboZY6zNZi7djjq2QaXPA4uwsO9jXf4AXk +S3_ENDPOINT=https://fsn1.your-objectstorage.com/ diff --git a/config.yaml b/config.yaml index 1419f67..b2ae806 100644 --- a/config.yaml +++ b/config.yaml @@ -18,3 +18,5 @@ app: ocr_crop_w_frac: 0.42 ocr_crop_h_frac: 0.22 thumb_max: 512 + exiftool_timeout_seconds: 15 + job_sleep_seconds: 1.0 diff --git a/docker-compose.yaml b/docker-compose.yaml index 5a3b5de..7b5f49f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,9 +5,28 @@ services: restart: unless-stopped environment: CONFIG_YAML: /app/config.yaml - DEBUG: "1" + DEBUG: "0" ports: - "5678:5678" volumes: - ./:/app - ./config.yaml:/app/config.yaml:ro + melesicu_shiny: + build: ./shiny-app + container_name: melesicu_shiny + restart: unless-stopped + environment: + PG_DSN: ${PG_DSN} + S3_BUCKET: ${S3_BUCKET} + ENTRANCE_PREFIX: "icu/entrance/" + PROCESSED_PREFIX: "icu/processed/" + THUMB_PREFIX: "icu/thumbnails/" + INVENTORY_TABLE: "cam_inventory" + DEPLOYMENT_TABLE: "cam_deployments" + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + AWS_DEFAULT_REGION: "us-east-1" + AWS_S3_ENDPOINT: ${S3_ENDPOINT} + AWS_S3_ENDPOINT_URL: ${S3_ENDPOINT} + ports: + - "3838:3838" diff --git a/main.py b/main.py index c0c0f20..facea1c 100644 --- a/main.py +++ b/main.py @@ -64,6 +64,8 @@ class AppConfig: ocr_crop_h_frac: float = 0.22 # bottom ~22% of height thumb_max: int = 512 + exiftool_timeout_seconds: int = 15 + job_sleep_seconds: float = 1.0 def load_config(path: str) -> AppConfig: @@ -92,6 +94,8 @@ def load_config(path: str) -> AppConfig: ocr_crop_w_frac=float(app.get("ocr_crop_w_frac", 0.42)), ocr_crop_h_frac=float(app.get("ocr_crop_h_frac", 0.22)), thumb_max=int(app.get("thumb_max", 512)), + exiftool_timeout_seconds=int(app.get("exiftool_timeout_seconds", 15)), + job_sleep_seconds=float(app.get("job_sleep_seconds", 1.0)), ) @@ -210,7 +214,7 @@ def _parse_ocr_datetime(*texts: str) -> datetime: raise ValueError(f"OCR timestamp not found. OCR1='{texts[0] if texts else ''}' OCR2='{texts[1] if len(texts) > 1 else ''}'") -def exif_write_with_exiftool(jpg_in: bytes, dt_original_local: datetime, dt_digitized_utc: datetime) -> bytes: +def exif_write_with_exiftool(jpg_in: bytes, dt_original_local: datetime, dt_digitized_utc: datetime, *, timeout_seconds: int) -> bytes: """ DateTimeOriginal = OCR time (camera local) + OffsetTimeOriginal DateTimeDigitized = servertime (UTC) + OffsetTimeDigitized=+00:00 @@ -237,7 +241,7 @@ def exif_write_with_exiftool(jpg_in: bytes, dt_original_local: datetime, dt_digi f"-OffsetTime={otd}", in_path, ] - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_seconds) with open(in_path, "rb") as f: return f.read() @@ -320,6 +324,41 @@ def job_set_status(cur, uuid: str, status: str, *, err: Optional[str] = None, pr """, (status, status, err, processed_key, thumb_key, uuid)) +def job_get(cur, uuid: str): + cur.execute(""" + SELECT status, processed_jpg_key, thumbnail_key, has_categories + FROM remote_cam.import_job + WHERE image_uuid = %s; + """, (uuid,)) + row = cur.fetchone() + if not row: + return None + return { + "status": row[0], + "processed_jpg_key": row[1], + "thumbnail_key": row[2], + "has_categories": row[3], + } + + +STATUS_ORDER = [ + "NEW", + "META_OK", + "OCR_OK", + "EXIF_OK", + "MOVED", + "THUMB_OK", + "CATEGORIES_OK", + "CLEANED", +] + + +def status_rank(status: Optional[str]) -> int: + if status in STATUS_ORDER: + return STATUS_ORDER.index(status) + return -1 + + def insert_resource(cur, uuid: str, typ: str): cur.execute(""" INSERT INTO remote_cam.resource(resource_uuid, type, import_ts) @@ -424,6 +463,16 @@ def update_ocr_ts(cur, uuid: str, ocr_ts: datetime): """, (ocr_ts, uuid)) +def get_ocr_ts(cur, uuid: str) -> Optional[datetime]: + cur.execute(""" + SELECT ocr_ts + FROM remote_cam.metadata + WHERE metadata_uuid = %s; + """, (uuid,)) + row = cur.fetchone() + return row[0] if row else None + + def insert_categories(cur, uuid: str, cat_json: dict): version = cat_json.get("version") cats = cat_json.get("categories") or [] @@ -528,54 +577,97 @@ def collect_ready_sets(s3: S3, cfg: AppConfig) -> list[EntranceSet]: def process_one(cur, s3: S3, cfg: AppConfig, s: EntranceSet): job_upsert_new(cur, s.uuid, s.jpg_key, s.meta_key, s.cat_key) + job = job_get(cur, s.uuid) or {} + status_now = job.get("status", "NEW") # 1) metadata.json -> DB meta = json.loads(s3.get_bytes(s.meta_key)) insert_metadata(cur, s.uuid, meta) insert_resource(cur, s.uuid, "metadata") - job_set_status(cur, s.uuid, "META_OK") + if status_rank("META_OK") >= status_rank(status_now): + job_set_status(cur, s.uuid, "META_OK") + status_now = "META_OK" # 2) JPG -> OCR bottom-right -> ocr_ts in CustomerTimezone jpg_bytes = s3.get_bytes(s.jpg_key) - customer_tzinfo, _ = parse_customer_tz(meta) - ocr_ts = ocr_extract_timestamp(jpg_bytes, cfg.ocr_crop_w_frac, cfg.ocr_crop_h_frac, customer_tzinfo) - update_ocr_ts(cur, s.uuid, ocr_ts) - insert_resource(cur, s.uuid, "image") - job_set_status(cur, s.uuid, "OCR_OK") + ocr_ts = None + if status_rank(status_now) < status_rank("OCR_OK"): + try: + customer_tzinfo, _ = parse_customer_tz(meta) + ocr_ts = ocr_extract_timestamp(jpg_bytes, cfg.ocr_crop_w_frac, cfg.ocr_crop_h_frac, customer_tzinfo) + update_ocr_ts(cur, s.uuid, ocr_ts) + insert_resource(cur, s.uuid, "image") + job_set_status(cur, s.uuid, "OCR_OK") + status_now = "OCR_OK" + except Exception as e: + job_set_status(cur, s.uuid, "OCR_FAIL", err=str(e)) + status_now = "OCR_FAIL" + elif status_rank(status_now) >= status_rank("OCR_OK"): + ocr_ts = get_ocr_ts(cur, s.uuid) # 3) EXIF write: # DateTimeOriginal = ocr_ts (local, keep offset tags) # DateTimeDigitized = servertime (UTC) - servertime = parse_servertime(meta) - jpg_exif = exif_write_with_exiftool(jpg_bytes, ocr_ts, servertime) - job_set_status(cur, s.uuid, "EXIF_OK") + jpg_exif = jpg_bytes + if status_rank(status_now) < status_rank("MOVED"): + if ocr_ts is None: + job_set_status(cur, s.uuid, "EXIF_SKIPPED", err="OCR missing") + status_now = "EXIF_SKIPPED" + else: + try: + servertime = parse_servertime(meta) + jpg_exif = exif_write_with_exiftool( + jpg_bytes, + ocr_ts, + servertime, + timeout_seconds=cfg.exiftool_timeout_seconds, + ) + job_set_status(cur, s.uuid, "EXIF_OK") + status_now = "EXIF_OK" + except Exception as e: + job_set_status(cur, s.uuid, "EXIF_FAIL", err=str(e)) + status_now = "EXIF_FAIL" # 4) Upload processed JPG (EXIF patched) - processed_key = f"{cfg.processed_prefix}{s.uuid}.jpg" - s3.put_bytes(processed_key, jpg_exif, "image/jpeg") - s3.head(processed_key) - job_set_status(cur, s.uuid, "MOVED", processed_key=processed_key) + processed_key = job.get("processed_jpg_key") or f"{cfg.processed_prefix}{s.uuid}.jpg" + if status_rank(status_now) < status_rank("MOVED"): + s3.put_bytes(processed_key, jpg_exif, "image/jpeg") + s3.head(processed_key) + job_set_status(cur, s.uuid, "MOVED", processed_key=processed_key) + status_now = "MOVED" # 5) Thumbnail - thumb_bytes = make_thumbnail_bytes(jpg_exif, cfg.thumb_max) - thumb_key = f"{cfg.thumb_prefix}{s.uuid}.jpg" - s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg") - s3.head(thumb_key) - job_set_status(cur, s.uuid, "THUMB_OK", thumb_key=thumb_key) + thumb_key = job.get("thumbnail_key") or f"{cfg.thumb_prefix}{s.uuid}.jpg" + if status_rank(status_now) < status_rank("THUMB_OK"): + if status_rank(status_now) >= status_rank("MOVED") and jpg_exif is jpg_bytes: + jpg_exif = s3.get_bytes(processed_key) + thumb_bytes = make_thumbnail_bytes(jpg_exif, cfg.thumb_max) + s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg") + s3.head(thumb_key) + job_set_status(cur, s.uuid, "THUMB_OK", thumb_key=thumb_key) + status_now = "THUMB_OK" # 6) Optional categories - if s.cat_key: + if s.cat_key and status_rank(status_now) < status_rank("CATEGORIES_OK"): cat_json = json.loads(s3.get_bytes(s.cat_key)) insert_categories(cur, s.uuid, cat_json) insert_resource(cur, s.uuid, "classification") job_set_status(cur, s.uuid, "CATEGORIES_OK") + status_now = "CATEGORIES_OK" # 7) Cleanup entrance only after verify - to_delete = [s.jpg_key, s.meta_key] - if s.cat_key: - to_delete.append(s.cat_key) - s3.delete_keys(to_delete) - job_set_status(cur, s.uuid, "CLEANED") + if status_rank(status_now) >= status_rank("THUMB_OK"): + try: + s3.head(processed_key) + s3.head(thumb_key) + to_delete = [s.jpg_key, s.meta_key] + if s.cat_key: + to_delete.append(s.cat_key) + s3.delete_keys(to_delete) + job_set_status(cur, s.uuid, "CLEANED") + status_now = "CLEANED" + except Exception as e: + job_set_status(cur, s.uuid, "CLEANUP_SKIPPED", err=str(e)) def run_once(cfg: AppConfig) -> int: @@ -585,10 +677,6 @@ def run_once(cfg: AppConfig) -> int: return 0 with psycopg.connect(cfg.pg_dsn) as db: - with db.cursor() as cur: - db_init(cur) - db.commit() - for s in ready: try: with db.transaction(): @@ -600,6 +688,8 @@ def run_once(cfg: AppConfig) -> int: job_upsert_new(cur, s.uuid, s.jpg_key, s.meta_key, s.cat_key) job_set_status(cur, s.uuid, "ERROR", err=str(e)) print(f"[ERROR] {s.uuid}: {e}") + if cfg.job_sleep_seconds > 0: + time.sleep(cfg.job_sleep_seconds) return len(ready) @@ -607,6 +697,11 @@ def main(): cfg_path = os.environ.get("CONFIG_YAML", "/app/config.yaml") cfg = load_config(cfg_path) + with psycopg.connect(cfg.pg_dsn) as db: + with db.cursor() as cur: + db_init(cur) + db.commit() + while True: n = run_once(cfg) if n == 0: diff --git a/shiny-app/Dockerfile b/shiny-app/Dockerfile new file mode 100644 index 0000000..caab08f --- /dev/null +++ b/shiny-app/Dockerfile @@ -0,0 +1,22 @@ +FROM rocker/shiny:latest + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + libxml2-dev \ + curl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ + && unzip awscliv2.zip \ + && ./aws/install \ + && rm -rf awscliv2.zip aws + +RUN R -e "install.packages(c('DBI','RPostgres','dplyr','DT','glue','shiny'), repos='https://cloud.r-project.org')" + +WORKDIR /srv/shiny-server +COPY app.R /srv/shiny-server/app.R + +EXPOSE 3838 diff --git a/shiny-app/app.R b/shiny-app/app.R new file mode 100644 index 0000000..5cd5433 --- /dev/null +++ b/shiny-app/app.R @@ -0,0 +1,517 @@ +library(shiny) +library(DBI) +library(RPostgres) +library(dplyr) +library(DT) +library(glue) + +pg_dsn <- Sys.getenv("PG_DSN", "") +s3_bucket <- Sys.getenv("S3_BUCKET", "") +s3_endpoint <- Sys.getenv("S3_ENDPOINT", Sys.getenv("AWS_S3_ENDPOINT", Sys.getenv("AWS_S3_ENDPOINT_URL", ""))) +aws_access_key <- Sys.getenv("S3_ACCESS_KEY", Sys.getenv("AWS_ACCESS_KEY_ID", "")) +aws_secret_key <- Sys.getenv("S3_SECRET_KEY", Sys.getenv("AWS_SECRET_ACCESS_KEY", "")) +aws_cli <- Sys.getenv("AWS_CLI", "aws") +thumb_prefix <- Sys.getenv("THUMB_PREFIX", "icu/thumbnails/") +processed_prefix <- Sys.getenv("PROCESSED_PREFIX", "icu/processed/") +entrance_prefix <- Sys.getenv("ENTRANCE_PREFIX", "icu/entrance/") +processed_prefix <- Sys.getenv("PROCESSED_PREFIX", "icu/processed/") +thumb_prefix <- Sys.getenv("THUMB_PREFIX", "icu/thumbnails/") + +inventory_table <- Sys.getenv("INVENTORY_TABLE", "") +deployment_table <- Sys.getenv("DEPLOYMENT_TABLE", "") + +if (pg_dsn == "") { + stop("PG_DSN env var is required") +} +if (s3_bucket == "") { + stop("S3_BUCKET env var is required") +} +if (s3_endpoint == "") { + stop("S3_ENDPOINT/AWS_S3_ENDPOINT env var is required") +} +if (aws_access_key == "" || aws_secret_key == "") { + stop("S3_ACCESS_KEY/S3_SECRET_KEY or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY required") +} + +Sys.setenv( + AWS_ACCESS_KEY_ID = aws_access_key, + AWS_SECRET_ACCESS_KEY = aws_secret_key, + AWS_EC2_METADATA_DISABLED = "true" +) + +parse_pg_dsn <- function(dsn) { + if (grepl("^postgresql://", dsn)) { + m <- regexec("^postgresql://([^:]+):([^@]+)@([^:/]+):?(\\d+)?/(.+)$", dsn) + r <- regmatches(dsn, m)[[1]] + if (length(r) == 0) stop("Invalid PG_DSN URL format") + return(list( + user = r[2], + password = r[3], + host = r[4], + port = ifelse(r[5] == "", 5432, as.integer(r[5])), + dbname = r[6] + )) + } + + if (grepl("host=", dsn)) { + parts <- strsplit(dsn, "\\s+")[[1]] + kv <- lapply(parts, function(p) strsplit(p, "=", fixed = TRUE)[[1]]) + vals <- setNames(lapply(kv, `[`, 2), lapply(kv, `[`, 1)) + return(list( + user = vals$user, + password = vals$password, + host = vals$host, + port = as.integer(vals$port), + dbname = vals$dbname + )) + } + + return(list(dbname = dsn)) +} + +pg <- parse_pg_dsn(pg_dsn) +pg_con <- do.call(dbConnect, c(list(RPostgres::Postgres()), pg)) +onStop(function() dbDisconnect(pg_con)) + +table_exists <- function(con, table_name) { + res <- dbGetQuery(con, "SELECT to_regclass($1) AS t", params = list(table_name)) + !is.na(res$t[1]) +} + +fetch_cameras <- function() { + if (inventory_table != "" && table_exists(pg_con, inventory_table)) { + q <- glue("SELECT DISTINCT cam_id FROM {inventory_table} ORDER BY 1") + return(dbGetQuery(pg_con, q)$cam_id) + } + character(0) +} + +fetch_projects <- function() { + if (deployment_table != "" && table_exists(pg_con, deployment_table)) { + q <- glue("SELECT DISTINCT project FROM {deployment_table} ORDER BY 1") + return(dbGetQuery(pg_con, q)$project) + } + character(0) +} + +build_query <- function(from_date, to_date, camera_id, project_name, limit_rows, offset_rows, use_date_filter) { + base <- " + SELECT + m.metadata_uuid, + m.import_ts, + m.ocr_ts, + m.\"CustomerTimezone\", + m.\"SignalStrength\", + m.\"Temperature\", + m.imei, + m.servertime, + ci.cam_id, + d.project, + c.category, + c.score + FROM remote_cam.metadata m + LEFT JOIN cam_inventory ci ON ci.imei = m.imei + LEFT JOIN LATERAL ( + SELECT d.project + FROM cam_deployments d + WHERE d.camera_id = ci.cam_id + AND m.servertime >= d.deployment_start + AND (d.deployment_end IS NULL OR m.servertime < d.deployment_end) + ORDER BY d.deployment_start DESC NULLS LAST + LIMIT 1 + ) d ON TRUE + LEFT JOIN LATERAL ( + SELECT category, score + FROM remote_cam.category c + WHERE c.image_uuid = m.metadata_uuid + ORDER BY score DESC NULLS LAST + LIMIT 1 + ) c ON TRUE + " + + params <- list() + + if (camera_id != "" && inventory_table != "" && table_exists(pg_con, inventory_table)) { + base <- paste0(base, " WHERE ci.cam_id = $", length(params) + 1) + params <- append(params, list(camera_id)) + } + + if (project_name != "" && deployment_table != "" && table_exists(pg_con, deployment_table)) { + prefix <- if (length(params) == 0) " WHERE " else " AND " + base <- paste0(base, prefix, " m.imei IN (SELECT ci2.imei FROM cam_inventory ci2 JOIN ", + deployment_table, " d ON d.camera_id = ci2.cam_id WHERE d.project = $", + length(params) + 1, ")") + params <- append(params, list(project_name)) + } + + if (camera_id == "" && project_name == "" && !use_date_filter) { + base <- paste0(base, " ORDER BY m.import_ts DESC LIMIT $", length(params) + 1, " OFFSET $", length(params) + 2) + params <- append(params, list(as.integer(limit_rows), as.integer(offset_rows))) + return(list(sql = base, params = params)) + } + + if (use_date_filter) { + prefix <- if (length(params) == 0) " WHERE " else " AND " + base <- paste0(base, prefix, " m.import_ts >= $", length(params) + 1, " AND m.import_ts < $", length(params) + 2) + params <- append(params, list(from_date, to_date)) + } + + base <- paste0(base, " ORDER BY m.import_ts DESC LIMIT $", length(params) + 1, " OFFSET $", length(params) + 2) + params <- append(params, list(as.integer(limit_rows), as.integer(offset_rows))) + list(sql = base, params = params) +} + +normalize_endpoint <- function(url) { + if (url == "") return("") + u <- url + if (!grepl("^https?://", u)) { + u <- paste0("https://", u) + } + sub("/+$", "", u) +} + +s3_endpoint <- normalize_endpoint(s3_endpoint) + +if (aws_access_key == "" || aws_secret_key == "" || s3_endpoint == "") { + stop("S3_ACCESS_KEY/S3_SECRET_KEY or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY and S3_ENDPOINT are required") +} + +normalize_endpoint <- function(url) { + if (url == "") return("") + u <- url + if (!grepl("^https?://", u)) { + u <- paste0("https://", u) + } + sub("/+$", "", u) +} + +s3_endpoint <- normalize_endpoint(s3_endpoint) +cache_dir <- Sys.getenv("THUMB_CACHE_DIR", "/tmp/thumb_cache") +dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) +addResourcePath("thumbs", cache_dir) + +presigned_url <- function(key, expires = 600) { + if (is.null(key) || is.na(key) || key == "") return(NULL) + key <- sub(paste0("^", s3_bucket, "/"), "", key) + cmd <- sprintf( + "\"%s\" --endpoint-url %s s3 presign s3://%s/%s --expires-in %d", + aws_cli, s3_endpoint, s3_bucket, key, as.integer(expires) + ) + tryCatch({ + out <- system(cmd, intern = TRUE) + if (length(out) == 0) return(NULL) + out[1] + }, error = function(e) { + message(sprintf("[DEBUG] presign error: %s", e$message)) + NULL + }) +} + +cache_thumbnail <- function(uuid) { + if (is.null(uuid) || is.na(uuid)) return(NULL) + local_path <- file.path(cache_dir, paste0(uuid, ".jpg")) + if (file.exists(local_path)) return(local_path) + key <- paste0(thumb_prefix, uuid, ".jpg") + cmd <- sprintf( + "\"%s\" --endpoint-url %s s3 cp s3://%s/%s \"%s\" --only-show-errors", + aws_cli, s3_endpoint, s3_bucket, key, local_path + ) + tryCatch({ + system(cmd, ignore.stdout = TRUE, ignore.stderr = TRUE) + if (file.exists(local_path)) return(local_path) + NULL + }, error = function(e) { + message(sprintf("[DEBUG] cache thumb error: %s", e$message)) + NULL + }) +} + +ui <- fluidPage( + titlePanel("MelesICU - Image Browser"), + tags$script(HTML(" + function melesThumbClick(el){ + var uuid = el.getAttribute('data-uuid'); + if(uuid){ + Shiny.setInputValue('thumb_click', uuid, {priority: 'event'}); + } + } + var melesScrollLock = false; + window.addEventListener('scroll', function() { + if (melesScrollLock) return; + var nearBottom = (window.innerHeight + window.scrollY) >= (document.body.offsetHeight - 200); + var canLoad = Shiny && Shiny.shinyapp && Shiny.shinyapp.$outputValues && Shiny.shinyapp.$outputValues.has_more_flag; + var isLoading = Shiny && Shiny.shinyapp && Shiny.shinyapp.$outputValues && Shiny.shinyapp.$outputValues.loading_flag; + if (nearBottom && canLoad && !isLoading) { + melesScrollLock = true; + Shiny.setInputValue('scroll_load', Date.now(), {priority: 'event'}); + setTimeout(function(){ melesScrollLock = false; }, 1000); + } + }); + ")), + sidebarLayout( + sidebarPanel( + dateRangeInput("date_range", "Datum", start = Sys.Date() - 7, end = Sys.Date()), + checkboxInput("use_date_filter", "Datumsfilter aktiv", value = TRUE), + selectInput("camera_id", "Kamera (cam_id)", choices = c("ALL", fetch_cameras()), selected = "ALL"), + selectInput("project_name", "Projekt", choices = c("ALL", fetch_projects()), selected = "ALL"), + numericInput("limit_rows", "Eintraege pro Seite", value = 20, min = 10, max = 2000, step = 10), + numericInput("page", "Seite", value = 1, min = 1, step = 1), + fluidRow( + column(6, actionButton("prev_page", "Zurueck")), + column(6, actionButton("next_page", "Vor")) + ), + actionButton("apply_filters", "Filter anwenden") + ), + mainPanel( + tags$style(HTML(" + .spinner { + margin: 20px 0; + width: 32px; + height: 32px; + border: 4px solid #ddd; + border-top-color: #333; + border-radius: 50%; + animation: spin 1s linear infinite; + } + @keyframes spin { to { transform: rotate(360deg); } } + ")), + conditionalPanel("output.loading_flag", tags$div(class = "spinner")), + uiOutput("debug_info"), + uiOutput("thumb_grid") + ) + ) +) + +server <- function(input, output, session) { + rv <- reactiveValues(data = NULL, selected = NULL, loading = FALSE, has_more = TRUE, debug = NULL) + cache_env <- new.env(parent = emptyenv()) + cache_keys <- character(0) + url_cache_env <- new.env(parent = emptyenv()) + url_cache_keys <- character(0) + + load_data <- function() { + from_date <- as.POSIXct(input$date_range[1], tz = "UTC") + to_date <- as.POSIXct(input$date_range[2] + 1, tz = "UTC") + cam <- ifelse(input$camera_id == "ALL", "", input$camera_id) + proj <- ifelse(input$project_name == "ALL", "", input$project_name) + page <- max(1, as.integer(input$page)) + limit <- as.integer(input$limit_rows) + offset <- (page - 1) * limit + + key <- paste(from_date, to_date, cam, proj, input$use_date_filter, limit, offset, sep = "|") + if (exists(key, envir = cache_env, inherits = FALSE)) { + rv$data <- get(key, envir = cache_env, inherits = FALSE) + return(invisible(NULL)) + } + + q <- build_query(from_date, to_date, cam, proj, limit, offset, input$use_date_filter) + rv$loading <- TRUE + on.exit({ rv$loading <- FALSE }, add = TRUE) + data <- dbGetQuery(pg_con, q$sql, params = q$params) + assign(key, data, envir = cache_env) + cache_keys <<- c(cache_keys, key) + if (length(cache_keys) > 20) { + drop <- head(cache_keys, length(cache_keys) - 20) + for (k in drop) rm(list = k, envir = cache_env) + cache_keys <<- tail(cache_keys, 20) + } + rv$data <- data + rv$has_more <- nrow(data) >= limit + + # Batch prefetch first 10 thumbnails in parallel + if (!is.null(data) && nrow(data) > 0) { + uuids <- head(data$metadata_uuid, 10) + tryCatch({ + parallel::mclapply(uuids, cache_thumbnail, mc.cores = 4) + }, error = function(e) { + message(sprintf("[DEBUG] prefetch error: %s", e$message)) + }) + } + + if (!is.null(data) && nrow(data) > 0) { + sample_uuid <- data$metadata_uuid[1] + thumb_key <- paste0(thumb_prefix, sample_uuid, ".jpg") + url <- presigned_url(thumb_key) + rv$debug <- list( + count = nrow(data), + sample_uuid = sample_uuid, + thumb_key = thumb_key, + url = url + ) + message(sprintf("[DEBUG] count=%d sample_uuid=%s thumb_key=%s url=%s", + nrow(data), sample_uuid, thumb_key, ifelse(is.null(url), "NULL", url))) + } else { + rv$debug <- list(count = 0) + message("[DEBUG] count=0") + } + } + + observeEvent(input$apply_filters, { + if (rv$loading) { + showNotification("Noch am Laden – bitte warten.", type = "message") + return(NULL) + } + load_data() + }, ignoreInit = TRUE) + + observeEvent(input$prev_page, { + if (rv$loading) return(NULL) + new_page <- max(1, as.integer(input$page) - 1) + updateNumericInput(session, "page", value = new_page) + load_data() + }, ignoreInit = TRUE) + + observeEvent(input$next_page, { + if (rv$loading) return(NULL) + new_page <- as.integer(input$page) + 1 + updateNumericInput(session, "page", value = new_page) + load_data() + }, ignoreInit = TRUE) + + observeEvent(input$scroll_load, { + if (rv$loading || !rv$has_more) return(NULL) + new_page <- as.integer(input$page) + 1 + updateNumericInput(session, "page", value = new_page) + load_data() + }, ignoreInit = TRUE) + + observeEvent(TRUE, { + if (rv$loading) return(NULL) + load_data() + }, once = TRUE) + + output$loading_flag <- reactive({ rv$loading }) + outputOptions(output, "loading_flag", suspendWhenHidden = FALSE) + + output$has_more_flag <- reactive({ rv$has_more }) + outputOptions(output, "has_more_flag", suspendWhenHidden = FALSE) + + format_cet <- function(ts) { + if (is.null(ts) || is.na(ts)) return("-") + format(as.POSIXct(ts, tz = "CET"), "%Y-%m-%d %H:%M:%S %Z") + } + + format_ts <- function(ts) { + if (is.null(ts) || is.na(ts)) return("-") + format(as.POSIXct(ts, tz = "CET"), "%Y-%m-%d %H:%M:%S %Z") + } + + format_diff <- function(server_ts, camera_ts) { + if (is.null(server_ts) || is.na(server_ts) || is.null(camera_ts) || is.na(camera_ts)) return("-") + secs <- as.numeric(difftime(server_ts, camera_ts, units = "secs")) + sign <- ifelse(secs >= 0, "+", "-") + secs <- abs(secs) + hh <- floor(secs / 3600) + mm <- floor((secs %% 3600) / 60) + ss <- floor(secs %% 60) + sprintf("%s%02d:%02d:%02d", sign, hh, mm, ss) + } + + output$thumb_grid <- renderUI({ + dat <- rv$data + if (is.null(dat) || nrow(dat) == 0) { + return(tags$div("Keine Ergebnisse.")) + } + + cards <- lapply(seq_len(nrow(dat)), function(i) { + row <- dat[i, ] + local_path <- cache_thumbnail(row$metadata_uuid) + if (is.null(local_path)) return(NULL) + url <- paste0("thumbs/", basename(local_path)) + tags$div( + style = "display:inline-block; width: 220px; margin: 6px; vertical-align: top;", + tags$div(class = "ph", style = "width: 220px; height: 220px; background:#eee; display:block;"), + tags$img( + src = url, + style = "width: 220px; height: 220px; object-fit: cover; display:block; cursor: pointer;", + loading = "lazy", + onload = "this.parentNode.querySelector('.ph').style.display='none';", + `data-uuid` = row$metadata_uuid, + onclick = "melesThumbClick(this)" + ), + tags$div(style = "font-size: 12px; color: #222;", + glue("CamID: {row$cam_id}")), + tags$div(style = "font-size: 12px; color: #444;", + glue("Serverzeit: {format_cet(row$servertime)}")), + tags$div(style = "font-size: 12px; color: #444;", + glue("Kamerazeit: {format_ts(row$ocr_ts)}")), + tags$div(style = "font-size: 12px; color: #444;", + glue("AI classification: {row$category} ({row$score})")), + { + delta <- format_diff(row$servertime, row$ocr_ts) + delta_secs <- if (delta == "-") NA else as.numeric(difftime(row$servertime, row$ocr_ts, units = "secs")) + is_warn <- !is.na(delta_secs) && abs(delta_secs) > 600 + style <- if (is_warn) "font-size: 12px; color: #c00; font-weight: bold;" else "font-size: 12px; color: #444;" + tags$div(style = style, glue("Delta: {delta}")) + } + ) + }) + + tags$div(cards) + }) + + output$debug_info <- renderUI({ + if (is.null(rv$debug)) return(NULL) + cnt <- rv$debug$count + if (is.null(cnt)) return(NULL) + if (cnt == 0) { + return(tags$div(style = "font-size:12px; color:#a00; margin-bottom:8px;", + "Debug: 0 Treffer.")) + } + tags$div(style = "font-size:12px; color:#555; margin-bottom:8px;", + glue("Debug: Treffer={cnt}, Sample UUID={rv$debug$sample_uuid}"), + tags$br(), + glue("Thumb-Key: {rv$debug$thumb_key}"), + tags$br(), + if (!is.null(rv$debug$url)) tags$a(href = rv$debug$url, "Sample Thumbnail (Link)", target = "_blank") else "Sample URL: NULL") + }) + + observeEvent(input$thumb_click, { + uuid <- input$thumb_click + dat <- rv$data + if (is.null(dat)) return(NULL) + row <- dat[dat$metadata_uuid == uuid, ] + if (nrow(row) != 1) return(NULL) + full_key <- paste0(processed_prefix, row$metadata_uuid, ".jpg") + presigned_cached <- function(key, expires = 600) { + now <- Sys.time() + if (exists(key, envir = url_cache_env, inherits = FALSE)) { + entry <- get(key, envir = url_cache_env, inherits = FALSE) + if (is.list(entry) && !is.null(entry$expires_at) && now < entry$expires_at) { + return(entry$url) + } + } + u <- presigned_url(key, expires = expires) + if (!is.null(u)) { + assign(key, list(url = u, expires_at = now + expires - 30), envir = url_cache_env) + url_cache_keys <<- c(url_cache_keys, key) + if (length(url_cache_keys) > 200) { + drop <- head(url_cache_keys, length(url_cache_keys) - 200) + for (k in drop) rm(list = k, envir = url_cache_env) + url_cache_keys <<- tail(url_cache_keys, 200) + } + } + u + } + + url <- presigned_cached(full_key) + if (is.null(url)) { + showModal(modalDialog( + title = glue("Bild {uuid}"), + "Full image nicht gefunden.", + easyClose = TRUE, + footer = NULL + )) + return(NULL) + } + showModal(modalDialog( + title = glue("Bild {uuid}"), + tags$img(src = url, style = "max-width: 100%; height: auto;"), + easyClose = TRUE, + footer = modalButton("Schließen"), + size = "l" + )) + }) +} + +shinyApp(ui, server)