amber-backend/pb_hooks/status.pb.js

/// <reference path="../pb_data/types.d.ts" />

// Operator status dashboard — the data behind pb_public/status.html.
//
//   GET /api/status                          (superuser) → one JSON snapshot
//   GET /api/status/logs?app=<uuid>&lines=N  (superuser) → a service's container logs
//
// WHY THIS EXISTS: answering "is Amber healthy?" meant opening Coolify's UI,
// clicking through per-app pages for status, clicking again for logs, then
// querying PocketBase separately for what the family is actually being served.
// This puts the five questions that get asked when something breaks on one screen:
// is it up, did the last deploy succeed, what did it say, are clients erroring,
// and is the newest release the one I think it is.
//
// **Superuser-gated, not user-gated.** There is no "operator" flag on `users` and
// inventing one would need a migration; the owner already has superuser
// credentials for /_/, so that is the honest gate. `nsfwEnabled` was considered
// and rejected — it is a content flag, and using it as an admin flag would mean
// any adult-enabled family member could read infrastructure logs.
//
// **This is the first hook in the project to make an outbound HTTP request.**
// Everything before it read only PocketBase's own database (see the note in
// amber-app/CLAUDE.md about the JSVM having no outbound-HTTP precedent). Two
// consequences, both handled below: every external call is wrapped so a dead
// dependency degrades one panel instead of failing the whole response, and every
// call carries a short timeout so a hung upstream cannot pin a PocketBase worker.
//
// **The Coolify token never reaches the browser.** That is the entire reason this
// endpoint exists rather than the page calling Coolify directly.
//
// The variable is COOLIFY_**READ**_TOKEN, and the name is the point: this dashboard
// only ever performs GETs, so it must never be handed the write-capable token that
// `coolify.sh` uses for deploys. A full token here would mean a PocketBase
// vulnerability could deploy or stop every app on the PaaS.
//
// NOTE: PocketBase runs each routerAdd handler in its own isolated JSVM, so
// anything a handler needs must be declared INSIDE it — file-scope helpers throw
// ReferenceError at request time. Nothing is hoisted here; keep it that way.

// Pretty URL, following the same pattern as /settings → /settings.html.
//
// Without it, `/status` does NOT 404 — PocketBase's `indexFallback` (on by
// default) quietly serves index.html for any unmatched static path, so the URL
// looked like it worked and returned the family landing page instead. The docs
// were written with the pretty URL, so the docs were wrong until this existed.
routerAdd("GET", "/status", (e) => e.redirect(302, "/status.html"))

routerAdd("GET", "/api/status", (e) => {
  // ---- helpers, all handler-local (isolated JSVM) ------------------------
  const safe = (label, fn) => {
    try {
      return fn()
    } catch (err) {
      return { error: String(err && err.message ? err.message : err), _label: label }
    }
  }

  const fetchJson = (url, headers, timeoutSec) => {
    const res = $http.send({
      url: url,
      method: "GET",
      headers: headers || {},
      timeout: timeoutSec || 6,
    })
    if (res.statusCode < 200 || res.statusCode >= 300) {
      throw new Error("HTTP " + res.statusCode)
    }
    return res.json
  }

  const env = (k) => {
    const v = $os.getenv(k)
    return v && v.length ? v : null
  }

  // ---- what the family is being served (local DB, always available) ------
  const releases = safe("releases", () => {
    // Newest build per platform. Three rows in practice; a 200-row window is
    // plenty and bounds the query regardless of history size.
    //
    // Filtered to variant='clean' rather than reading whatever is newest: rows
    // from the 18+ flavour are still in this collection, and a leftover one
    // could otherwise be reported as what the family is being served.
    const rows = $app.findRecordsByFilter(
      "releases", "variant = 'clean'", "-buildNumber", 200, 0)
    const newest = {}
    for (const r of rows) {
      const key = r.getString("platform")
      if (newest[key]) continue
      newest[key] = {
        platform: r.getString("platform"),
        version: r.getString("version"),
        build: r.getInt("buildNumber"),
        size: r.getInt("size"),
        sha8: r.getString("sha256").substring(0, 8),
        published: r.getDateTime("created").string(),
      }
    }
    return Object.keys(newest).sort().map((k) => newest[k])
  })

  // ---- are clients erroring (local DB) ----------------------------------
  const clients = safe("clients", () => {
    const since = new DateTime()
    // 24h back. dateTime arithmetic in the JSVM is awkward, so compare on a
    // string built from the epoch — client_logs.created is an autodate.
    const cutoff = new DateTime(
      new Date(Date.now() - 24 * 3600 * 1000).toISOString().replace("T", " ").substring(0, 19)
    )
    const rows = $app.findRecordsByFilter(
      "client_logs",
      "created >= {:cutoff}",
      "-created",
      2000,
      0,
      { cutoff: cutoff }
    )
    const byEvent = {}
    let errors = 0
    let stutter = 0
    for (const r of rows) {
      const ev = r.getString("event") || "?"
      byEvent[ev] = (byEvent[ev] || 0) + 1
      const kind = (r.getString("kind") || "").toLowerCase()
      if (kind === "error" || ev === "uncaught" || ev === "player_error") errors++
      // A viewer saying it stuttered while the counters saw nothing is the
      // documented blind spot worth surfacing, not burying in a total.
      const meta = r.getString("meta") || ""
      if (meta.indexOf('"rating":"stutter"') >= 0) stutter++
    }
    return { window: "24h", total: rows.length, errors: errors, stutterReports: stutter, byEvent: byEvent }
  })

  const accounts = safe("accounts", () => {
    const rows = $app.findRecordsByFilter("users", "id != ''", "", 500, 0)
    return { total: rows.length }
  })

  // ---- containers (Coolify) --------------------------------------------
  // Degrades to configured:false rather than failing the response, so the panels
  // above still render on a box where these env vars were never set.
  const coolify = safe("coolify", () => {
    const base = env("COOLIFY_URL")
    const token = env("COOLIFY_READ_TOKEN")
    if (!base || !token) {
      return {
        configured: false,
        why: "COOLIFY_URL / COOLIFY_READ_TOKEN are not set on this container — add them " +
             "in Coolify → amber-backend → Environment Variables and redeploy.",
      }
    }
    const apps = fetchJson(base.replace(/\/+$/, "") + "/api/v1/applications",
                           { Authorization: "Bearer " + token }, 8)
    // Only Amber's own services; the same instance also hosts unrelated apps.
    const mine = [
      "amber-backend", "amber-api", "stremio_prehrajto", "Stremio_Torbox",
      "adult-stremio-addon", "myanime-pair-server", "stripchat-proxy",
    ]
    const out = []
    for (const a of apps) {
      const name = a.name || ""
      let keep = false
      for (const m of mine) if (name.indexOf(m) === 0) keep = true
      if (!keep) continue
      out.push({
        name: name,
        uuid: a.uuid,
        status: a.status || "?",
        fqdn: a.fqdn || null,
        // sync.petruzalekr.cz is kSyncServerBase — Amber-critical despite the
        // legacy repo name. Flagged so nobody decommissions it as old myAnime.
        note: name.indexOf("myanime-pair-server") === 0 ? "kSyncServerBase — Amber-critical" : null,
      })
    }
    out.sort((x, y) => (x.name < y.name ? -1 : 1))
    return { configured: true, apps: out }
  })

  // ---- amber-api vitals -----------------------------------------------
  const amberApi = safe("amberApi", () => {
    const base = env("AMBER_API_URL")
    const secret = env("AMBER_ADMIN_SECRET")
    if (!base || !secret) {
      return {
        configured: false,
        why: "AMBER_API_URL / AMBER_ADMIN_SECRET are not set on this container. " +
             "ADMIN_SECRET is the operator role (x-amber-admin), NOT one of the " +
             "SERVICE_SECRETS pairs, which grant an addon's identity.",
      }
    }
    const stats = fetchJson(base.replace(/\/+$/, "") + "/v1/admin/stats",
                            { "x-amber-admin": secret }, 8)
    return { configured: true, probe: stats.probe, queue: stats.queue,
             metadata: stats.metadata, health: stats.health,
             // Upstream liveness rides along on the same call rather than a
             // second round trip: it is produced by the same service and read
             // by the same screen.
             upstreams: stats.upstreams,
             throughput: stats.throughput }
  })

  return e.json(200, {
    generatedAt: new DateTime().string(),
    releases: releases,
    clients: clients,
    accounts: accounts,
    coolify: coolify,
    amberApi: amberApi,
  })
}, $apis.requireSuperuserAuth())

// Error history, sliced by the things that explain it: which publish, which OS,
// which flavour.
//
//   GET /api/status/errors?days=14   (superuser)
//
// **Rate, not raw count, is what attributes errors to a publish.** Versions have
// wildly different exposure — at the time of writing 1.1.4+13 had 177 records to
// 1.1.5+14's 20 — so raw counts make whichever build the family used most look
// worst. Every version row therefore carries total, errors and errors/total.
//
// **Flavour is not recorded yet.** The app's TelemetryService sends appVersion,
// platform and device, but nothing carries clean-vs-adult. This reads a top-level
// `flavor` field first and then `meta.flavor`, so the day the app starts sending
// it this endpoint needs no change; until then the column honestly reads unknown.
//
// The window is capped by retention, not by choice: client_logs.pb.js trims to 14
// days on a nightly cron, so nothing older exists to graph.
routerAdd("GET", "/api/status/errors", (e) => {
  let days = parseInt(e.request.url.query().get("days") || "14", 10)
  if (!(days > 0) || days > 14) days = 14   // 14 = the retention ceiling

  const ROW_CAP = 5000
  const cutoffMs = Date.now() - days * 24 * 3600 * 1000
  const cutoff = new DateTime(
    new Date(cutoffMs).toISOString().replace("T", " ").substring(0, 19)
  )

  let rows = []
  try {
    rows = $app.findRecordsByFilter(
      "client_logs", "created >= {:cutoff}", "-created", ROW_CAP, 0, { cutoff: cutoff }
    )
  } catch (err) {
    return e.json(500, { error: String(err && err.message ? err.message : err) })
  }

  const isError = (r) => {
    const kind = (r.getString("kind") || "").toLowerCase()
    const ev = r.getString("event") || ""
    return kind === "error" || ev === "uncaught" || ev === "player_error" || ev === "exo_error"
  }
  const days_ = {}, byVersion = {}, byPlatform = {}, byEvent = {}
  // Pre-seed every day so the graph has no gaps — a missing bar and a zero bar
  // mean different things and must not look alike.
  for (let i = 0; i < days; i++) {
    const d = new Date(Date.now() - i * 24 * 3600 * 1000).toISOString().substring(0, 10)
    days_[d] = { date: d, total: 0, errors: 0 }
  }

  const bump = (bag, key, err) => {
    if (!bag[key]) bag[key] = { key: key, total: 0, errors: 0 }
    bag[key].total++
    if (err) bag[key].errors++
  }

  for (const r of rows) {
    const err = isError(r)
    const day = r.getDateTime("created").string().substring(0, 10)
    if (days_[day]) { days_[day].total++; if (err) days_[day].errors++ }
    bump(byVersion, r.getString("appVersion") || "?", err)
    bump(byPlatform, r.getString("platform") || "?", err)
    if (err) {
      const ev = r.getString("event") || "?"
      byEvent[ev] = (byEvent[ev] || 0) + 1
    }
  }

  const rate = (o) => Object.assign({}, o, {
    rate: o.total ? Math.round((o.errors / o.total) * 1000) / 10 : 0,
  })
  const list = (bag) => Object.keys(bag).map((k) => rate(bag[k]))
                              .sort((a, b) => b.total - a.total)

  return e.json(200, {
    window: days,
    truncated: rows.length >= ROW_CAP,
    sampled: rows.length,
    // Oldest first, so the graph reads left-to-right as time.
    days: Object.keys(days_).sort().map((k) => days_[k]),
    byVersion: list(byVersion),
    byPlatform: list(byPlatform),
    topErrorEvents: Object.keys(byEvent).sort((a, b) => byEvent[b] - byEvent[a])
                          .slice(0, 8).map((k) => ({ key: k, errors: byEvent[k] })),
  })
}, $apis.requireSuperuserAuth())

// Container logs for one service, proxied so the Coolify token stays server-side.
//
// Runtime logs ARE available from the Coolify API — /api/v1/applications/<uuid>/logs
// returns live container stdout. An earlier design assumed this needed SSH to the
// VPS; it does not, which is how the cinema-rip bug was found in amber-api's log.
routerAdd("GET", "/api/status/logs", (e) => {
  const app = e.request.url.query().get("app")
  if (!app || !/^[a-z0-9]{10,40}$/.test(app)) {
    return e.json(400, { error: "app must be a Coolify application uuid" })
  }
  let lines = parseInt(e.request.url.query().get("lines") || "200", 10)
  if (!(lines > 0) || lines > 1000) lines = 200

  const base = $os.getenv("COOLIFY_URL")
  const token = $os.getenv("COOLIFY_READ_TOKEN")
  if (!base || !token) {
    return e.json(503, { error: "COOLIFY_URL / COOLIFY_READ_TOKEN not set on this container" })
  }
  try {
    const res = $http.send({
      url: base.replace(/\/+$/, "") + "/api/v1/applications/" + app + "/logs?lines=" + lines,
      method: "GET",
      headers: { Authorization: "Bearer " + token },
      timeout: 15,
    })
    if (res.statusCode < 200 || res.statusCode >= 300) {
      return e.json(502, { error: "coolify HTTP " + res.statusCode })
    }
    return e.json(200, { app: app, lines: lines, logs: (res.json && res.json.logs) || "" })
  } catch (err) {
    return e.json(502, { error: String(err && err.message ? err.message : err) })
  }
}, $apis.requireSuperuserAuth())