Review follow-ups: durable alert acceptance, credential epochs, fatal privilege drop

README.md

@@ -249,8 +249,8 @@ first start:
 sysmon-web -agent-names sysmon-web.example.net
 ```
 
-**2. Make a token for the site.** Open **Admin -> Monitoring boxes ->
-Add a box**. Give it a site name and a label. The page then shows the
+**2. Make a token for the site.** Open **Admin -> Agents & alerters ->
+Add credential**. Give it a site name and a label. The page then shows the
 complete set of config lines, with the token in them, and a button that
 copies them. The server keeps only a hash, so it shows the token one time.
 

android/app/src/main/java/com/sysmon/app/Models.kt

@@ -165,6 +165,11 @@ data class TestPushResponse(val status: String = "", val warning: String? = null
 
 @Serializable
 data class HistoryEvent(
+    // The store's immutable sequence number; row identity for lists.
+    // Timestamps only carry second precision, so two same-status alerts
+    // in one second would collide without it. 0 on rows from servers
+    // that predate the field.
+    val id: Long = 0,
     val timestamp: String = "",
     @SerialName("object_name") val objectName: String = "",
     @SerialName("local_name") val localName: String = "",

android/app/src/main/java/com/sysmon/app/ui/HistoryScreen.kt

@@ -39,7 +39,7 @@ import kotlinx.coroutines.launch
 
 private enum class HistoryFilter(val label: String) {
     ALL("ALL"),
-    DOWNS("DOWNS"),
+    DOWNS("PROBLEMS"),
     RECOVERIES("RECOVERIES")
 }
 
@@ -177,7 +177,10 @@ fun HistoryScreen() {
                     else "No events match the filter"
                 )
             }
-            else -> itemsIndexed(filtered) { _, ev ->
+            else -> itemsIndexed(
+                filtered,
+                key = { idx, ev -> if (ev.id != 0L) ev.id else "row-" + idx }
+            ) { _, ev ->
                 HistoryRow(ev, clock)
             }
         }
@@ -226,20 +229,35 @@ private fun HistoryRow(ev: HistoryEvent, clock: Long) {
                         color = MaterialTheme.colorScheme.onSurfaceVariant
                     )
                 }
+                // The alert's own message (an alerter's text), when there
+                // is one - it is the payload, not decoration.
+                if (ev.description.isNotEmpty()) {
+                    Text(
+                        text = ev.description,
+                        style = MaterialTheme.typography.bodySmall,
+                        color = MaterialTheme.colorScheme.onSurfaceVariant,
+                        maxLines = 2,
+                        overflow = TextOverflow.Ellipsis
+                    )
+                }
                 Row(
                     verticalAlignment = Alignment.CenterVertically,
                     horizontalArrangement = Arrangement.spacedBy(6.dp)
                 ) {
-                    Text(
-                        text = ev.prevStatus,
-                        style = MaterialTheme.typography.labelMedium,
-                        color = statusColor(ev.prevStatus)
-                    )
-                    Text(
-                        text = "→",
-                        style = MaterialTheme.typography.labelMedium,
-                        color = MaterialTheme.colorScheme.onSurfaceVariant
-                    )
+                    // An object's first-ever event has no previous state:
+                    // no empty badge, no dangling arrow.
+                    if (ev.prevStatus.isNotEmpty()) {
+                        Text(
+                            text = ev.prevStatus,
+                            style = MaterialTheme.typography.labelMedium,
+                            color = statusColor(ev.prevStatus)
+                        )
+                        Text(
+                            text = "→",
+                            style = MaterialTheme.typography.labelMedium,
+                            color = MaterialTheme.colorScheme.onSurfaceVariant
+                        )
+                    }
                     Text(
                         text = ev.newStatus,
                         style = MaterialTheme.typography.labelMedium,

docs/ALERTERS.md

@@ -14,14 +14,20 @@ its own "Alerters" section; the config editor and the map never see it.
 
 ## Getting a token
 
-Same as a monitoring box: **Admin -> Monitoring boxes -> Add a box**.
-Mint a token under the name the alerter will use (letters, digits,
-`-`, `_`; max 64 chars). The name is the alerter's identity - it
-appears in notifications and on the Fleet page - so name the thing,
-not the machine: `backupd`, not `server3`.
+**Admin -> Agents & alerters -> Add credential**, with the credential
+type set to **External alerter** - the panel then shows the greeting
+line below instead of sysmond config. Mint the token under the name the
+alerter will use (letters, digits, `-`, `_`; max 64 chars). The name
+is the alerter's identity - it appears in notifications and on the
+Fleet page - so name the thing, not the machine: `backupd`, not
+`server3`.
 
-Revoking the token on the same page cuts the alerter off at its next
-connection attempt.
+The type is part of the credential: a token minted for an alerter is
+refused if something greets with it as a sysmond, and the other way
+around.
+
+Revoking the token on the same page cuts the alerter off immediately -
+the live connection is closed and the next attempt is refused.
 
 ## Connecting
 
@@ -40,9 +46,12 @@ connection attempt.
 ## Protocol
 
 Text lines, terminated by `\n` (a trailing `\r` is tolerated). One
-line may carry at most 4096 bytes; anything past that on the same
-line is discarded, not buffered. Every reply is one line starting
-`333 ` (success) or `444 ` (refusal).
+line may carry at most 4096 bytes; a longer line is refused with
+`444 line too long` rather than processed as something shorter than
+what was sent. After authentication the connection survives the
+refusal; an overlong (or otherwise malformed) **greeting** gets the
+444 and then the socket closes, like any other failed handshake.
+Every reply is one line starting `333 ` (success) or `444 ` (refusal).
 
 ### Handshake (first line, within 20 seconds of connecting)
 
@@ -51,9 +60,10 @@ line is discarded, not buffered. Every reply is one line starting
 - `333 welcome` - authenticated; send alerts from here on.
 - `444 rejected` - bad name/token pair, or the token is revoked. The
   socket closes; back off before retrying.
-- `444 this token belongs to a sysmond` - the token was minted for (and
-  first used by) a monitoring box; a token keeps the kind of its first
-  handshake forever. Mint a separate token for the alerter.
+- `444 this token belongs to a sysmond` - the token was minted for a
+  monitoring box. New credentials are permanently typed when minted;
+  only legacy records with no stored kind are claimed by their first
+  successful greeting. Mint a separate alerter credential.
 
 Everything after the token is what the application calls itself -
 free text up to 128 characters, e.g. `Bacula 15.0 nightly backups`.
@@ -75,9 +85,31 @@ does the talking.
   plain "name reports object STATUS" is generated.
 - Reply is `333 ok` once accepted, or `444 <reason>` for a malformed
   line. A `444` never closes the connection; fix the line and carry on.
-- `444 busy - ...` means the server's delivery pipeline is backed up
-  and the alert was **not** accepted. Retry the same line after a short
-  delay; `333 ok` is the only reply that means the alert was taken.
+- `444 could not record the alert - ...` means the history write
+  failed and the alert was **not** accepted; retry after a short
+  delay. `444 alert history unavailable ...` means this server cannot
+  record alerts at all. `333 ok` is the only reply that means the
+  alert was taken.
+What `333 ok` promises, exactly: the alert is recorded in the web
+UI's **alert history** - written to disk before the reply, visible on
+the History page, surviving server restarts - and, when push is
+enabled, queued for immediate phone delivery in order with everything
+else this alerter has sent. The history record is the delivery
+guarantee; push is the extra channel on top, attempted only after the
+history commit. Phone-side delivery is best-effort and its failures
+never appear on the wire (a refusal would invite a retry that
+duplicates the recorded event): a provider outage after acceptance
+shows in the server log and the admin Push Log, a saturated push
+queue in the server log. If an alert matters, keep re-sending
+transitions as the condition changes rather than treating one 333 as
+the end of the story.
+
+Ingestion is rate limited per alerter: a burst of 30 alerts, refilling
+at one per second. Past that, `444 rate limited - ...` refuses the
+line before anything is recorded - the alert history is shared with
+the fleet's host transitions and bounded, and a looping script must
+not be able to churn it. Back off and retry; a well-behaved alerter
+sending state *transitions* never notices this limit.
 
 Semantics, identical to a sysmond's transitions:
 
@@ -88,9 +120,10 @@ Semantics, identical to a sysmond's transitions:
   replaces the earlier alert on the phones rather than stacking a
   second notification, because `<alerter>:<object>` is the collapse
   key, exactly as host alerts collapse per host.
-- Delivery honors the master push switch in the admin UI; alerts sent
-  while push is disabled are acknowledged and dropped, and the server
-  log says so.
+- The master push switch in the admin UI only governs the phones:
+  alerts sent while push is disabled are still accepted, recorded, and
+  shown in the web UI - they just page nobody, and the server log says
+  so.
 
 ### Keepalive and goodbye
 
@@ -114,17 +147,22 @@ logs, the registry - so renaming a nickname never re-keys anything.
 
 ## What the web UI does with alerts
 
+- Records each alert in the **History** page's log, alongside host
+  transitions, as `<alerter>:<object>` - with the status it changed
+  from and how long the previous state lasted, once this server has
+  seen the object before.
 - Push notifications to every subscribed phone, with the priority
-  routing above.
+  routing above, when push is enabled.
 - The admin **Push Log** records each fan-out like any other.
 - The **Fleet page** shows the alerter: connected or gone, what it
   shows as (nickname or application name), its address, how many
   alerts it has sent, and the last one.
 
-Alerts are fire-and-forget by design: they are not stored as host
-state, do not appear on the dashboard, and are not replayed to phones
-that subscribe later. If a thing needs its state *tracked*, it wants
-to be a monitored host on a sysmond, not an alerter.
+Alerts are events, not tracked state: they do not appear on the
+dashboard's host board and are not replayed to phones that subscribe
+later. If a thing needs its state *tracked* - polled, colored,
+acknowledged - it wants to be a monitored host on a sysmond, not an
+alerter.
 
 ## Example: shell
 
@@ -143,17 +181,25 @@ to be a monitored host on a sysmond, not an alerter.
 ## Example: Python
 
 ```python
-import socket, ssl, time
+import os, socket, ssl, time
 
 HOST, PORT = "sysmon-web.example.net", 1347
-NAME, TOKEN = "backupd", "tok-abc123..."
-
+NAME = "backupd"
+# A credential stays out of source and argv: a mode-0600 file or the
+# environment of the service unit that runs this.
+TOKEN = os.environ["SYSMON_TOKEN"]
+
+# Verify the certificate AND its name. sysmon-web puts the names the
+# daemons dial it by into its generated certificate (the -agent-names
+# flag); start it with the name you use here in that list. Only fall
+# back to ctx.check_hostname = False against an old certificate that
+# carries no usable name - it weakens the check to "any holder of a
+# CA-signed cert", so regenerate the certificate instead if you can.
 ctx = ssl.create_default_context(cafile="aggregator-ca.pem")
-ctx.check_hostname = False  # self-signed cert carries no hostname
 
 def connect():
     raw = socket.create_connection((HOST, PORT), timeout=20)
-    tls = ctx.wrap_socket(raw)
+    tls = ctx.wrap_socket(raw, server_hostname=HOST)
     f = tls.makefile("rw", newline="\n")
     f.write(f"ALERTER {NAME} {TOKEN} Bacula 15.0 nightly backups\n"); f.flush()
     if not f.readline().startswith("333"):

docs/WEB_DEPLOYMENT.md

@@ -22,14 +22,15 @@ returns your shell prompt.
 
 | Invocation | Behaviour |
 |---|---|
-| `sysmon-web …` | Daemonizes, **silent** (no logs). |
+| `sysmon-web …` | Daemonizes, **silent** (nothing is watching stderr). |
 | `sysmon-web -debug …` | Stays in the foreground, logs to **stderr**. Use this to find out why something won't start. |
-| `sysmon-web -foreground …` | Stays in the foreground, still silent. For process supervisors that track the PID themselves (systemd `Type=simple`, OpenBSD `rc.d`). Add `-debug` to also get logs. |
+| `sysmon-web -foreground …` | Stays in the foreground, logs to **stderr**. For process supervisors that track the PID themselves (systemd `Type=simple`, OpenBSD `rc.d`) - normal warnings and errors land in the journal / rc log, where an operator can find them. |
 
-Logs are **off unless `-debug`** is given - a daemon shouldn't chatter.
-If the service won't come up, the move is always: stop it, run it once
-in the foreground with `-debug`, read the error, fix, restart under the
-supervisor.
+Under a supervisor, ordinary logs are always on: a monitoring server
+that drops a page must not also drop the log line saying so. Only the
+self-daemonized mode is silent, because its stderr goes nowhere. If
+the service won't come up, run it once with `-debug`, read the error,
+fix, restart under the supervisor.
 
 > Under a supervisor you almost always want `-foreground`. If you let it
 > self-daemonize under `Type=simple`, systemd sees the parent exit
@@ -90,10 +91,13 @@ Use the shipped unit (`web-ui/sysmon-web.service`); the important bits:
 Type=simple
 User=www-data
 Group=www-data
-# www-data can't mkdir under root-owned /var/www, so create the socket
-# dir as root first (the "+" runs these as root despite User=www-data).
+# The binary prepares its own directories only when it starts as root;
+# under User=www-data it never does, so the unit prepares every path
+# the unprivileged process cannot create (the "+" runs these as root).
 ExecStartPre=+/bin/mkdir -p /var/www/run
 ExecStartPre=+/bin/chown www-data:www-data /var/www/run
+ExecStartPre=+/usr/bin/install -d -o www-data -g www-data /var/backups/sysmon
+ExecStartPre=+/bin/sh -c 'touch /var/log/sysmon-web-audit.log && chown www-data:www-data /var/log/sysmon-web-audit.log'
 ExecStart=/usr/local/bin/sysmon-web \
   -foreground \
   -socket /var/www/run/sysmon-web.sock \
@@ -105,9 +109,11 @@ Restart=always
 
 Because it runs as `www-data`, the socket is already owned by nginx's
 user - no `-socket-*` flags needed. The `ExecStartPre` lines create
-`/var/www/run` (which `ProtectSystem=strict` also lists in
-`ReadWritePaths`). Add `-debug` to the `ExecStart` line temporarily to
-get logs in the journal (`journalctl -u sysmon-web -f`).
+the socket directory, the backup directory, and the audit log, all
+owned by `www-data` (`/var/lib/sysmon` is `StateDirectory=`, which
+systemd itself prepares). With `-foreground`, normal logs already go
+to the journal (`journalctl -u sysmon-web -f`); `-debug` adds verbose
+diagnostics on top.
 
 ```sh
 cp web-ui/sysmon-web.service /etc/systemd/system/

ios/Sysmon/HistoryView.swift

@@ -1,7 +1,7 @@
 import SwiftUI
 
 // Alert history: every host state transition the server has observed,
-// newest first, with All / Downs / Recoveries filtering.
+// newest first, with All / Problems / Recoveries filtering.
 struct HistoryView: View {
     @EnvironmentObject var session: Session
     @State private var events: [HistoryEvent] = []
@@ -17,7 +17,7 @@ struct HistoryView: View {
 
     enum HistoryFilter: String, CaseIterable {
         case all = "All"
-        case downs = "Downs"
+        case downs = "Problems"
         case recoveries = "Recoveries"
     }
 
@@ -128,20 +128,32 @@ struct HistoryRow: View {
                         .font(.system(size: 11))
                         .foregroundColor(Theme.subtle)
                 }
+                // The alert's own message (an alerter's text) is the
+                // payload - show it, not just the transition.
+                if let desc = event.description, !desc.isEmpty {
+                    Text(desc)
+                        .font(.system(size: 11))
+                        .foregroundColor(Theme.subtle)
+                        .lineLimit(2)
+                }
                 HStack(spacing: 6) {
-                    Text(event.prevStatus)
-                        .font(.system(size: 9, weight: .bold))
-                        .tracking(0.5)
-                        .foregroundColor(statusColor(event.prevStatus))
-                    Image(systemName: "arrow.right")
-                        .font(.system(size: 8, weight: .semibold))
-                        .foregroundColor(Theme.faint)
+                    // An object's first-ever event has no previous state:
+                    // no empty badge, no dangling arrow.
+                    if !event.prevStatus.isEmpty {
+                        Text(event.prevStatus)
+                            .font(.system(size: 9, weight: .bold))
+                            .tracking(0.5)
+                            .foregroundColor(statusColor(event.prevStatus))
+                        Image(systemName: "arrow.right")
+                            .font(.system(size: 8, weight: .semibold))
+                            .foregroundColor(Theme.faint)
+                    }
                     Text(event.newStatus)
                         .font(.system(size: 9, weight: .bold))
                         .tracking(0.5)
                         .foregroundColor(statusColor(event.newStatus))
                     Spacer()
-                    if let dur = event.prevDuration, dur > 0 {
+                    if let dur = event.prevDuration, dur > 0, !event.prevStatus.isEmpty {
                         Text("was \(event.prevStatus.lowercased()) \(formatUptime(dur))")
                             .font(.system(size: 10))
                             .foregroundColor(Theme.subtle)

ios/Sysmon/MainView.swift

@@ -266,9 +266,15 @@ struct HostRow: View {
             StatusDot(status: host.overallStatus, pulse: host.isDown && !host.isPaused)
             VStack(alignment: .leading, spacing: 3) {
                 HStack(spacing: 6) {
+                    // A long hostname ellipsizes rather than wrapping or
+                    // squeezing the tags out of the row; layoutPriority
+                    // makes the badges yield space before the name does.
                     Text(host.hostname)
                         .font(.system(size: 15, weight: .semibold))
                         .foregroundColor(Theme.ink)
+                        .lineLimit(1)
+                        .truncationMode(.tail)
+                        .layoutPriority(1)
                     if !host.siteTag.isEmpty {
                         SiteTag(name: host.siteTag)
                     }