diff --git a/src/lib/journald.ts b/src/lib/journald.ts index 5079819b1..4d83ece4d 100644 --- a/src/lib/journald.ts +++ b/src/lib/journald.ts @@ -21,7 +21,7 @@ export interface SpawnJournalctlOpts { unit?: string; containerId?: string; format: string; - filterString?: string; + filter?: string | string[]; since?: string; until?: string; } @@ -57,8 +57,16 @@ export function spawnJournalctl(opts: SpawnJournalctlOpts): ChildProcess { args.push('-o'); args.push(opts.format); - if (opts.filterString) { - args.push(opts.filterString); + if (opts.filter != null) { + // A single filter argument without spaces can be passed as a string + if (typeof opts.filter === 'string') { + args.push(opts.filter); + } else { + // Multiple filter arguments need to be passed as an array of strings + // instead of a single string with spaces, as `spawn` will interpret + // the single string as a single argument to journalctl, which is invalid. + args.push(...opts.filter); + } } log.debug('Spawning journalctl', args.join(' ')); diff --git a/src/logger.ts b/src/logger.ts index da52f948f..ce181e92a 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -127,6 +127,7 @@ export function logSystemMessage( ); } } +export type LogSystemMessage = typeof logSystemMessage; export function lock(containerId: string): Bluebird.Disposer<() => void> { return takeGlobalLockRW(containerId).disposer((release) => { diff --git a/src/logging/monitor.ts b/src/logging/monitor.ts index f70c5f20f..d0782fb39 100644 --- a/src/logging/monitor.ts +++ b/src/logging/monitor.ts @@ -1,8 +1,9 @@ import { pipeline } from 'stream/promises'; +import { setTimeout } from 'timers/promises'; import { spawnJournalctl, toJournalDate } from '../lib/journald'; import log from '../lib/supervisor-console'; -import { setTimeout } from 'timers/promises'; +import type { LogSystemMessage } from '../logger'; export type MonitorHook = (message: { message: string; @@ -18,6 +19,7 @@ interface JournalRow { MESSAGE: string | number[]; PRIORITY: string; __REALTIME_TIMESTAMP: string; + _SYSTEMD_UNIT?: string; } // Wait 5s when journalctl failed before trying to read the logs again @@ -55,6 +57,9 @@ async function* splitStream(chunkIterable: AsyncIterable) { * Streams logs from journalctl and calls container hooks when a record is received matching container id */ class LogMonitor { + // Additional host services we want to stream the logs for + private HOST_SERVICES = ['os-power-mode.service', 'os-fan-profile.service']; + private containers: { [containerId: string]: { hook: MonitorHook; @@ -65,18 +70,23 @@ class LogMonitor { // Only stream logs since the start of the supervisor private lastSentTimestamp = Date.now() - performance.now(); - public async start(): Promise { + public async start(logSystemMessage: LogSystemMessage): Promise { try { // TODO: do not spawn journalctl if logging is not enabled const { stdout, stderr } = spawnJournalctl({ all: true, follow: true, format: 'json', - filterString: '_SYSTEMD_UNIT=balena.service', + filter: [ + // Monitor logs from balenad by default for container log-streaming + 'balena.service', + // Add any host services we want to stream + ...this.HOST_SERVICES, + ].map((s) => `_SYSTEMD_UNIT=${s}`), since: toJournalDate(this.lastSentTimestamp), }); if (!stdout) { - // this will be catched below + // This error will be caught below throw new Error('failed to open process stream'); } @@ -96,6 +106,8 @@ class LogMonitor { self.containers[row.CONTAINER_ID_FULL] ) { await self.handleRow(row); + } else if (self.HOST_SERVICES.includes(row._SYSTEMD_UNIT)) { + await self.handleHostServiceRow(row, logSystemMessage); } } catch { // ignore parsing errors @@ -116,7 +128,7 @@ class LogMonitor { `Spawning another process to watch host service logs in ${wait / 1000}s`, ); await setTimeout(wait); - void this.start(); + void this.start(logSystemMessage); } public isAttached(containerId: string): boolean { @@ -157,6 +169,23 @@ class LogMonitor { await this.containers[containerId].hook({ message, isStdErr, timestamp }); this.lastSentTimestamp = timestamp; } + + private async handleHostServiceRow( + row: JournalRow & { _SYSTEMD_UNIT: string }, + logSystemMessage: LogSystemMessage, + ) { + const message = messageFieldToString(row.MESSAGE); + if (message == null) { + return; + } + // Prune '.service' from the end of the service name + void logSystemMessage( + `${row._SYSTEMD_UNIT.replace(/\.service$/, '')}: ${message}`, + {}, + undefined, + false, + ); + } } const logMonitor = new LogMonitor(); diff --git a/src/supervisor.ts b/src/supervisor.ts index e6581543a..c17c3b0ce 100644 --- a/src/supervisor.ts +++ b/src/supervisor.ts @@ -84,7 +84,7 @@ export class Supervisor { apiBinder.start(), ]); - await logMonitor.start(); + await logMonitor.start(logger.logSystemMessage); } } diff --git a/test/unit/lib/journald.spec.ts b/test/unit/lib/journald.spec.ts index 34c97ea55..b0eb3052f 100644 --- a/test/unit/lib/journald.spec.ts +++ b/test/unit/lib/journald.spec.ts @@ -28,6 +28,7 @@ describe('lib/journald', () => { unit: 'nginx.service', containerId: 'abc123', format: 'json-pretty', + filter: ['_SYSTEMD_UNIT=test.service', '_SYSTEMD_UNIT=test2.service'], since: '2014-03-25 03:59:56.654563', until: '2014-03-25 03:59:59.654563', }); @@ -48,6 +49,8 @@ describe('lib/journald', () => { '2014-03-25 03:59:56.654563', '-U', '2014-03-25 03:59:59.654563', + '_SYSTEMD_UNIT=test.service', + '_SYSTEMD_UNIT=test2.service', ]; const actualCommand = spawn.firstCall.args[0];