Merge pull request #156 from mountaindude/master

v5.6.0

.codeclimate.yml

@@ -39,3 +39,6 @@ checks:
   identical-code:
     config:
       threshold: # language-specific defaults. an override will affect all languages.
+
+exclude_patterns:
+  - "**/node_modules/"

.editorconfig

@@ -0,0 +1,22 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = space
+end_of_line = lf
+charset = utf-8
+indent_size = 4
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false
+insert_final_newline = false
+
+[{,test/}{actual,fixtures}/**]
+trim_trailing_whitespace = false
+insert_final_newline = false
+
+[templates/**]
+trim_trailing_whitespace = false
+insert_final_newline = false

.gitignore

@@ -13,6 +13,7 @@ ssl/*
 *.tmp
 
 # Logs
+log/
 logs
 *.log
 npm-debug.log*

changelog.md

@@ -2,6 +2,30 @@
 
 Releases are [available on Github](https://github.com/ptarmiganlabs/butler-sos/releases).
 
+## 5.6.0
+
+### New features
+
+1. Added exclude lists for user sessions. If some user accounts (such as system accounts) should not be included in the tracking of user sessions, just add them to the exclude list in the config file. ([#62](https://github.com/ptarmiganlabs/butler/issues/62))
+2. Copy user activity tracking ("user events") concept from Butler. That feature belongs to the domain handled by Butler SOS, so it's moved here instead. Future development will happen here rather than in the Butler project. ([#147](https://github.com/ptarmiganlabs/butler/issues/147)
+3. Added user exclude list to user events config. [#151](https://github.com/ptarmiganlabs/butler/issues/151)
+4. Added the ability to specify arbitrary tags that is attached to the user event metrics when stored in InfluxDB. [#153](https://github.com/ptarmiganlabs/butler/issues/153))
+5. Anonymous telemtry added. The data included in the telemetry data is a) what kinf of computer (Windows/Linux, Node.js version etc) Butler SOS runs on, and b) which features are enabled. That's it - no identifiable data what so ever, and of course no actual metrics. ([#148](https://github.com/ptarmiganlabs/butler/issues/148))
+6. Documented the dependency on InfluxDB 1.x, rather than the most recent version 2.x. The latest version is awesome but has some breaking changes which cause it to fail when used with Butler SOS. ([#152](https://github.com/ptarmiganlabs/butler/issues/152))
+
+### Fixes and patches
+
+1. Dependencies updated to stay sharp and secure.
+2. Fixed some typos, made some log texts more descriptive. ([#149](https://github.com/ptarmiganlabs/butler/issues/149) and more)
+3. Replaced the end-of-life Moment.js library with Luxon.js. ([#150](https://github.com/ptarmiganlabs/butler/issues/150))
+4. A bit saner logging in Butler log files when some Sense servers in a cluster were not described in Butler SOS config file. ([#155](https://github.com/ptarmiganlabs/butler/issues/155))
+
+### Changed behavior and/or breaking changes
+
+1. The Butler SOS config file has not been very consistent when it comes to using "enable", "enabled" and variants thereof. Confusing. With this version we're cleaning up some of those variants.  
+   Going forward "enable" is used in Butler SOS (and also in more and more of the other [Butler tools](https://github.com/ptarmiganlabs). Most of the cases fixed in this release, some remain as todos.  
+   Using the [latest config file syntax](/docs/reference/config_file_format/) is **strongly** recommended, but efforts have been made to ensure that the old "enabled" variants also work for time being.). ([#154](https://github.com/ptarmiganlabs/butler/issues/154))
+
 ## 5.5.3
 
 - Dependencies updated to stay sharp and secure.

docs/log4net_user-audit-event/LocalLogConfig.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <appender name="EventSession" type="log4net.Appender.UdpAppender">
+        <filter type="log4net.Filter.StringMatchFilter">
+            <param name="stringToMatch" value="Start session for user" />
+        </filter>
+        <filter type="log4net.Filter.StringMatchFilter">
+            <param name="stringToMatch" value="Stop session for user" />
+        </filter>
+        <filter type="log4net.Filter.DenyAllFilter" />
+        <param name="remoteAddress" value="<FQDN or IP of server where Butler is running>" />
+        <param name="remotePort" value="9997" />
+        <param name="encoding" value="utf-8" />
+        <layout type="log4net.Layout.PatternLayout">
+            <converter>
+                <param name="name" value="hostname" />
+                <param name="type" value="Qlik.Sense.Logging.log4net.Layout.Pattern.HostNamePatternConverter" />
+            </converter>
+            <param name="conversionpattern" value="/proxy-session/;%hostname;%property{Command};%property{UserDirectory};%property{UserId};%property{Origin};%property{Context};%message" />
+        </layout>
+    </appender>
+
+    <appender name="EventConnection" type="log4net.Appender.UdpAppender">
+        <filter type="log4net.Filter.StringMatchFilter">
+            <param name="stringToMatch" value="connection Opened for session" />
+        </filter>
+        <filter type="log4net.Filter.StringMatchFilter">
+            <param name="stringToMatch" value="connection Closed for session" />
+        </filter>
+        <filter type="log4net.Filter.DenyAllFilter" />
+        <param name="remoteAddress" value="<FQDN or IP of server where Butler is running>" />
+        <param name="remotePort" value="9997" />
+        <param name="encoding" value="utf-8" />
+        <layout type="log4net.Layout.PatternLayout">
+            <converter>
+                <param name="name" value="hostname" />
+                <param name="type" value="Qlik.Sense.Logging.log4net.Layout.Pattern.HostNamePatternConverter" />
+            </converter>
+            <param name="conversionpattern" value="/proxy-connection/;%hostname;%property{Command};%property{UserDirectory};%property{UserId};%property{Origin};%property{Context};%message" />
+        </layout>
+    </appender>
+
+    <logger name="AuditActivity.Proxy">
+        <appender-ref ref="EventSession" />
+        <appender-ref ref="EventConnection" />
+    </logger>
+</configuration>

src/.eslintrc.yml

@@ -2,6 +2,7 @@ parserOptions:
   ecmaVersion: 2020
 env:
   node: true
+  es6: true
 extends: 'eslint:recommended'
 rules:
   indent:
@@ -23,6 +24,3 @@ rules:
     - warn
   no-unused-vars:
     - warn
-
-
-

src/butler-sos.js

@@ -9,16 +9,19 @@ const sessionMetrics = require('./lib/sessionmetrics');
 const appNamesExtract = require('./lib/appnamesextract');
 const heartbeat = require('./lib/heartbeat');
 const serviceUptime = require('./lib/service_uptime');
+const udp = require('./lib/udp_handlers');
+const telemetry = require('./lib/telemetry');
 
 globals.initInfluxDB();
 
-if (globals.config.get('Butler-SOS.uptimeMonitor.enabled') == true) {
+if ((globals.config.has('Butler-SOS.uptimeMonitor.enabled') && globals.config.get('Butler-SOS.uptimeMonitor.enabled') == true) || 
+    (globals.config.has('Butler-SOS.uptimeMonitor.enable') && globals.config.get('Butler-SOS.uptimeMonitor.enable') == true)) {
     serviceUptime.serviceUptimeStart();
 }
 
 mainScript();
 
-function mainScript() {
+async function mainScript() {
     // Load certificates to use when connecting to healthcheck API
     var path = require('path'),
         certFile = path.resolve(__dirname, globals.config.get('Butler-SOS.cert.clientCert')),
@@ -50,37 +53,83 @@ function mainScript() {
     );
 
     // Set up heartbeats, if enabled in the config file
-    if (globals.config.get('Butler-SOS.heartbeat.enabled') == true) {
+    if ((globals.config.has('Butler-SOS.heartbeat.enabled') && globals.config.get('Butler-SOS.heartbeat.enabled') == true) || 
+        (globals.config.has('Butler-SOS.heartbeat.enable') && globals.config.get('Butler-SOS.heartbeat.enable') == true)) {
         heartbeat.setupHeartbeatTimer(globals.config, globals.logger);
     }
 
-    // Set specific log level (if/when needed to override the config file setting)
-    // Possible values are { error: 0, warn: 1, info: 2, verbose: 3, debug: 4, silly: 5 }
-    // Default is to use log level defined in config file
-    globals.logger.info('--------------------------------------');
-    globals.logger.info('Starting Butler SOS');
-    globals.logger.info(`Log level: ${globals.getLoggingLevel()}`);
-    globals.logger.info(`App version: ${globals.appVersion}`);
-    globals.logger.info('--------------------------------------');
-
-    // Log info about what Qlik Sense certificates are being used
-    globals.logger.debug(`Client cert: ${certFile}`);
-    globals.logger.debug(`Client cert key: ${keyFile}`);
-    globals.logger.debug(`CA cert: ${caFile}`);
+    try {
+        // Get host info
+        globals.hostInfo = await globals.initHostInfo();
+        globals.logger.debug('CONFIG: Initiated host info data structures');
+
+        // Set specific log level (if/when needed to override the config file setting)
+        // Possible values are { error: 0, warn: 1, info: 2, verbose: 3, debug: 4, silly: 5 }
+        // Default is to use log level defined in config file
+        globals.logger.info('--------------------------------------');
+        globals.logger.info('Starting Butler SOS');
+        globals.logger.info(`Log level: ${globals.getLoggingLevel()}`);
+        globals.logger.info(`App version: ${globals.appVersion}`);
+        globals.logger.info('');
+        globals.logger.info(`Node version   : ${globals.hostInfo.node.nodeVersion}`);
+        globals.logger.info(`Architecture   : ${globals.hostInfo.si.os.arch}`);
+        globals.logger.info(`Platform       : ${globals.hostInfo.si.os.platform}`);
+        globals.logger.info(`Release        : ${globals.hostInfo.si.os.release}`);
+        globals.logger.info(`Distro         : ${globals.hostInfo.si.os.distro}`);
+        globals.logger.info(`Codename       : ${globals.hostInfo.si.os.codename}`);
+        globals.logger.info(`Virtual        : ${globals.hostInfo.si.system.virtual}`);
+        globals.logger.info(`Processors     : ${globals.hostInfo.si.cpu.processors}`);
+        globals.logger.info(`Physical cores : ${globals.hostInfo.si.cpu.physicalCores}`);
+        globals.logger.info(`Cores          : ${globals.hostInfo.si.cpu.cores}`);
+        globals.logger.info(`Docker arch.   : ${globals.hostInfo.si.cpu.hypervizor}`);
+        globals.logger.info(`Total memory   : ${globals.hostInfo.si.memory.total}`);
+        globals.logger.info('--------------------------------------');
+
+        // Log info about what Qlik Sense certificates are being used
+        globals.logger.info(`Client cert: ${certFile}`);
+        globals.logger.info(`Client cert key: ${keyFile}`);
+        globals.logger.info(`CA cert: ${caFile}`);
+
+        // Set up anon usage reports, if enabled
+        if (
+            globals.config.has('Butler-SOS.anonTelemetry') == false ||
+            (globals.config.has('Butler-SOS.anonTelemetry') == true && globals.config.get('Butler-SOS.anonTelemetry') == true)
+        ) {
+            telemetry.setupAnonUsageReportTimer();
+            globals.logger.verbose('MAIN: Anonymous telemetry reporting has been set up.');
+        }
+    } catch (err) {
+        globals.logger.error(`CONFIG: Error initiating host info: ${err}`);
+    }
 
     // ---------------------------------------------------
+    // Set up UDP handler
+    if (globals.config.has('Butler-SOS.userEvents.enable') && globals.config.get('Butler-SOS.userEvents.enable')) {
+        udp.udpInitUserActivityServer();
 
+        globals.logger.debug(`MAIN: Server for UDP server: ${globals.udpServer.host}`);
+
+        // Start UDP server for user activity events
+        globals.udpServer.userActivitySocket.bind(
+            globals.udpServer.portUserActivity,
+            globals.udpServer.host,
+        );
+    }
+
+    // ---------------------------------------------------
     // Start Docker healthcheck REST server on port set in config file
-    if (globals.config.get('Butler-SOS.dockerHealthCheck.enabled') == true) {
+    if ((globals.config.has('Butler-SOS.dockerHealthCheck.enabled') && globals.config.get('Butler-SOS.dockerHealthCheck.enabled') == true) || 
+        (globals.config.has('Butler-SOS.dockerHealthCheck.enable') && globals.config.get('Butler-SOS.dockerHealthCheck.enable') == true)) {
         globals.logger.verbose('MAIN: Starting Docker healthcheck server...');
 
         restServer.listen(globals.config.get('Butler-SOS.dockerHealthCheck.port'), function () {
             globals.logger.info('MAIN: Docker healthcheck server now listening');
         });
-    };
+    }
 
     // Set up extraction of data from log db
-    if (globals.config.get('Butler-SOS.logdb.enableLogDb') == true) {
+    if ((globals.config.has('Butler-SOS.logdb.enableLogDb') && globals.config.get('Butler-SOS.logdb.enableLogDb') == true) || 
+        (globals.config.has('Butler-SOS.logdb.enable') && globals.config.get('Butler-SOS.logdb.enable') == true)) {
         logDb.setupLogDbTimer();
     }