From 1ba9426993cc29e5fd7a5ba90fb495359681efa1 Mon Sep 17 00:00:00 2001 From: Chelsea Wright Date: Thu, 13 Aug 2026 14:50:43 +0200 Subject: [PATCH] feat(data_library): add journald sample package Adds data_library/journald/journal_json.log, 16 `journalctl -o json` records across the syslog, stdout, journal, kernel and driver transports: sshd auth success/failure/lockout, nginx worker exit and upstream refusal, systemd unit failure, sudo (authorized and NOT-in-sudoers), a UFW block, an OOM kill, cron, dockerd, postgres, and journald rate-limit suppression. Fields use raw journald names (_PID, _EXE, MESSAGE, PRIORITY, SYSLOG_IDENTIFIER, _HOSTNAME) rather than OpenTelemetry-conventional ones, which is what the OTel journald receiver actually delivers. journald's canonical timestamp __REALTIME_TIMESTAMP is epoch microseconds and has no ctime directive, so those values are static; syslog-transport records also carry SYSLOG_TIMESTAMP, which does use a directive and advances per emission. Both noted in the filegen docs. --- docs/generator/filegen.md | 16 ++++++++++++++++ .../data_library/journald/journal_json.log | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 generator/filegen/embeddedlibrary/data_library/journald/journal_json.log diff --git a/docs/generator/filegen.md b/docs/generator/filegen.md index 3d1fb94..f83430d 100644 --- a/docs/generator/filegen.md +++ b/docs/generator/filegen.md @@ -54,6 +54,22 @@ Fri Jan 14 08:22:17 2026 nginx.webserver.test nginx: 10.20.30.40 - - [Fri Jan 14 Mon Jan 13 10:15:23 2026 checkpoint.firewall.test Check Point: orig=192.168.1.100 Rule=allow_http Action=Accept Protocol=tcp src=192.168.1.100 dst=203.0.113.1 sport=54321 dport=80 ``` +### Journald (`journalctl -o json`) + +The `journald` package emits one JSON object per line in the shape the +OpenTelemetry `journald` receiver consumes — raw journald field names +(`_PID`, `_EXE`, `MESSAGE`, `PRIORITY`, `SYSLOG_IDENTIFIER`, `_HOSTNAME`), +not OpenTelemetry-conventional names. + +``` +{"__REALTIME_TIMESTAMP":"1783087320123456","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_IDENTIFIER":"sshd","SYSLOG_TIMESTAMP":"Jan 13 15:30:45","PRIORITY":"6","_COMM":"sshd","_EXE":"/usr/sbin/sshd","_PID":"1842","_SYSTEMD_UNIT":"ssh.service","MESSAGE":"Accepted publickey for deploy from 10.0.4.12 port 51322 ssh2: ED25519 SHA256:abcd1234"} +``` + +Note: journald's canonical timestamp, `__REALTIME_TIMESTAMP`, is epoch +microseconds, which has no ctime directive — those values are static in the +sample. Records carried over the syslog transport also carry +`SYSLOG_TIMESTAMP`, which does use a directive and advances per emission. + ## Timestamp Directives The File generator automatically processes timestamp directives in log files, replacing them with actual formatted timestamps. Use these directives in your log files to enable dynamic timestamp generation: diff --git a/generator/filegen/embeddedlibrary/data_library/journald/journal_json.log b/generator/filegen/embeddedlibrary/data_library/journald/journal_json.log new file mode 100644 index 0000000..610f0d1 --- /dev/null +++ b/generator/filegen/embeddedlibrary/data_library/journald/journal_json.log @@ -0,0 +1,16 @@ +{"__REALTIME_TIMESTAMP":"1783087320123456","__MONOTONIC_TIMESTAMP":"884512300","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_FACILITY":"4","SYSLOG_IDENTIFIER":"sshd","SYSLOG_TIMESTAMP":"%b %d %H:%M:%S","PRIORITY":"6","_UID":"0","_GID":"0","_COMM":"sshd","_EXE":"/usr/sbin/sshd","_CMDLINE":"sshd: deploy [priv]","_PID":"1842","_SYSTEMD_UNIT":"ssh.service","_SYSTEMD_CGROUP":"/system.slice/ssh.service","MESSAGE":"Accepted publickey for deploy from 10.0.4.12 port 51322 ssh2: ED25519 SHA256:abcd1234"} +{"__REALTIME_TIMESTAMP":"1783087322987654","__MONOTONIC_TIMESTAMP":"884515164","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_FACILITY":"4","SYSLOG_IDENTIFIER":"sshd","SYSLOG_TIMESTAMP":"%b %d %H:%M:%S","PRIORITY":"4","_UID":"0","_GID":"0","_COMM":"sshd","_EXE":"/usr/sbin/sshd","_CMDLINE":"sshd: unknown [preauth]","_PID":"1849","_SYSTEMD_UNIT":"ssh.service","_SYSTEMD_CGROUP":"/system.slice/ssh.service","MESSAGE":"Failed password for invalid user admin from 203.0.113.45 port 44120 ssh2"} +{"__REALTIME_TIMESTAMP":"1783087324110022","__MONOTONIC_TIMESTAMP":"884516287","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_FACILITY":"4","SYSLOG_IDENTIFIER":"sshd","SYSLOG_TIMESTAMP":"%b %d %H:%M:%S","PRIORITY":"3","_UID":"0","_GID":"0","_COMM":"sshd","_EXE":"/usr/sbin/sshd","_CMDLINE":"sshd: unknown [preauth]","_PID":"1851","_SYSTEMD_UNIT":"ssh.service","_SYSTEMD_CGROUP":"/system.slice/ssh.service","MESSAGE":"error: maximum authentication attempts exceeded for invalid user admin from 203.0.113.45 port 44126 ssh2 [preauth]"} +{"__REALTIME_TIMESTAMP":"1783087330445566","__MONOTONIC_TIMESTAMP":"884522622","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"stdout","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"nginx","PRIORITY":"6","_UID":"33","_GID":"33","_COMM":"nginx","_EXE":"/usr/sbin/nginx","_CMDLINE":"nginx: worker process","_PID":"2210","_SYSTEMD_UNIT":"nginx.service","_SYSTEMD_CGROUP":"/system.slice/nginx.service","MESSAGE":"worker process 2233 exited with code 0"} +{"__REALTIME_TIMESTAMP":"1783087331778899","__MONOTONIC_TIMESTAMP":"884523955","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"stdout","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"nginx","PRIORITY":"3","_UID":"33","_GID":"33","_COMM":"nginx","_EXE":"/usr/sbin/nginx","_CMDLINE":"nginx: worker process","_PID":"2211","_SYSTEMD_UNIT":"nginx.service","_SYSTEMD_CGROUP":"/system.slice/nginx.service","MESSAGE":"connect() failed (111: Connection refused) while connecting to upstream, client: 10.0.4.88, server: api.example.com, request: \"GET /v1/orders HTTP/1.1\", upstream: \"http://10.0.5.20:8080/v1/orders\""} +{"__REALTIME_TIMESTAMP":"1783087340001122","__MONOTONIC_TIMESTAMP":"884532178","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"journal","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","PRIORITY":"6","_UID":"0","_GID":"0","_COMM":"systemd","_EXE":"/usr/lib/systemd/systemd","_CMDLINE":"/sbin/init","_PID":"1","_SYSTEMD_UNIT":"init.scope","_SYSTEMD_CGROUP":"/init.scope","MESSAGE":"Started Daily apt download activities."} +{"__REALTIME_TIMESTAMP":"1783087341334455","__MONOTONIC_TIMESTAMP":"884533511","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"journal","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","PRIORITY":"4","_UID":"0","_GID":"0","_COMM":"systemd","_EXE":"/usr/lib/systemd/systemd","_CMDLINE":"/sbin/init","_PID":"1","_SYSTEMD_UNIT":"init.scope","_SYSTEMD_CGROUP":"/init.scope","MESSAGE":"nginx.service: Main process exited, code=exited, status=1/FAILURE"} +{"__REALTIME_TIMESTAMP":"1783087342667788","__MONOTONIC_TIMESTAMP":"884534844","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"journal","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd","PRIORITY":"3","_UID":"0","_GID":"0","_COMM":"systemd","_EXE":"/usr/lib/systemd/systemd","_CMDLINE":"/sbin/init","_PID":"1","_SYSTEMD_UNIT":"init.scope","_SYSTEMD_CGROUP":"/init.scope","MESSAGE":"Failed to start A high performance web server and a reverse proxy server."} +{"__REALTIME_TIMESTAMP":"1783087350990011","__MONOTONIC_TIMESTAMP":"884543167","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_FACILITY":"10","SYSLOG_IDENTIFIER":"sudo","SYSLOG_TIMESTAMP":"%b %d %H:%M:%S","PRIORITY":"5","_UID":"1001","_GID":"1001","_COMM":"sudo","_EXE":"/usr/bin/sudo","_CMDLINE":"sudo systemctl restart nginx","_PID":"3401","_AUDIT_LOGINUID":"1001","_SYSTEMD_UNIT":"session-42.scope","_SYSTEMD_CGROUP":"/user.slice/user-1001.slice/session-42.scope","MESSAGE":"deploy : TTY=pts/0 ; PWD=/home/deploy ; USER=root ; COMMAND=/usr/bin/systemctl restart nginx"} +{"__REALTIME_TIMESTAMP":"1783087352223344","__MONOTONIC_TIMESTAMP":"884544400","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_FACILITY":"10","SYSLOG_IDENTIFIER":"sudo","SYSLOG_TIMESTAMP":"%b %d %H:%M:%S","PRIORITY":"4","_UID":"1002","_GID":"1002","_COMM":"sudo","_EXE":"/usr/bin/sudo","_CMDLINE":"sudo cat /etc/shadow","_PID":"3455","_AUDIT_LOGINUID":"1002","_SYSTEMD_UNIT":"session-43.scope","_SYSTEMD_CGROUP":"/user.slice/user-1002.slice/session-43.scope","MESSAGE":"contractor : user NOT in sudoers ; TTY=pts/2 ; PWD=/home/contractor ; USER=root ; COMMAND=/usr/bin/cat /etc/shadow"} +{"__REALTIME_TIMESTAMP":"1783087360556677","__MONOTONIC_TIMESTAMP":"884552733","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"kernel","SYSLOG_FACILITY":"0","SYSLOG_IDENTIFIER":"kernel","PRIORITY":"4","MESSAGE":"[UFW BLOCK] IN=eth0 OUT= MAC=00:16:3e:5f:2a:1b SRC=198.51.100.77 DST=10.0.4.21 LEN=60 TOS=0x00 PREC=0x00 TTL=52 ID=54321 DF PROTO=TCP SPT=51544 DPT=23 WINDOW=29200 RES=0x00 SYN URGP=0"} +{"__REALTIME_TIMESTAMP":"1783087361889900","__MONOTONIC_TIMESTAMP":"884554066","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"kernel","SYSLOG_FACILITY":"0","SYSLOG_IDENTIFIER":"kernel","PRIORITY":"3","MESSAGE":"Out of memory: Killed process 4820 (java) total-vm:8388608kB, anon-rss:6291456kB, file-rss:0kB, shmem-rss:0kB, UID:1003 pgtables:12288kB oom_score_adj:0"} +{"__REALTIME_TIMESTAMP":"1783087370112233","__MONOTONIC_TIMESTAMP":"884562289","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"syslog","SYSLOG_FACILITY":"9","SYSLOG_IDENTIFIER":"CRON","SYSLOG_TIMESTAMP":"%b %d %H:%M:%S","PRIORITY":"6","_UID":"0","_GID":"0","_COMM":"cron","_EXE":"/usr/sbin/cron","_CMDLINE":"/usr/sbin/cron -f","_PID":"918","_SYSTEMD_UNIT":"cron.service","_SYSTEMD_CGROUP":"/system.slice/cron.service","MESSAGE":"(root) CMD (/usr/local/bin/backup.sh >> /var/log/backup.log 2>&1)"} +{"__REALTIME_TIMESTAMP":"1783087380445566","__MONOTONIC_TIMESTAMP":"884572622","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"stdout","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"dockerd","PRIORITY":"6","_UID":"0","_GID":"0","_COMM":"dockerd","_EXE":"/usr/bin/dockerd","_CMDLINE":"/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock","_PID":"1155","_SYSTEMD_UNIT":"docker.service","_SYSTEMD_CGROUP":"/system.slice/docker.service","MESSAGE":"Container 3f9a2c1d8b7e stopped with exit code 137"} +{"__REALTIME_TIMESTAMP":"1783087390778899","__MONOTONIC_TIMESTAMP":"884582955","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"stdout","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"postgres","PRIORITY":"4","_UID":"113","_GID":"119","_COMM":"postgres","_EXE":"/usr/lib/postgresql/16/bin/postgres","_CMDLINE":"/usr/lib/postgresql/16/bin/postgres -D /var/lib/postgresql/16/main","_PID":"1290","_SYSTEMD_UNIT":"postgresql@16-main.service","_SYSTEMD_CGROUP":"/system.slice/system-postgresql.slice/postgresql@16-main.service","MESSAGE":"could not receive data from client: Connection reset by peer"} +{"__REALTIME_TIMESTAMP":"1783087400001122","__MONOTONIC_TIMESTAMP":"884592178","_BOOT_ID":"8f3c2a1b4d5e6f708192a3b4c5d6e7f8","_MACHINE_ID":"a1b2c3d4e5f60718293a4b5c6d7e8f90","_HOSTNAME":"linux-host01.test","_TRANSPORT":"driver","SYSLOG_FACILITY":"3","SYSLOG_IDENTIFIER":"systemd-journald","PRIORITY":"5","_UID":"0","_GID":"0","_COMM":"systemd-journal","_EXE":"/lib/systemd/systemd-journald","_CMDLINE":"/lib/systemd/systemd-journald","_PID":"412","_SYSTEMD_UNIT":"systemd-journald.service","_SYSTEMD_CGROUP":"/system.slice/systemd-journald.service","MESSAGE":"Suppressed 1247 messages from ssh.service"}