From 735f4159fb0b05931e9bd7b72dda66de3f2d28ea Mon Sep 17 00:00:00 2001 From: Jan Kaliszewski Date: Thu, 5 Oct 2023 06:11:00 +0200 Subject: [PATCH] STOMP+SSL-related changes: security fixes + convenience enhancement (+ doc updates/improvements) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit doc: a few fixes/improvements ad STOMP bots and *n6* feed lib, bots, doc: only cosmetic/very minor tweaks and comments lib, bots, pkg, doc: drop support for `stomp.py` older than 4.1.12 The affected bots are: *STOMP collector* (`StompCollectorBot` defined in `intelmq.bots.collectors.stomp.collector`) and *STOMP output* (`StompOutputBot` defined in `intelmq.bots.outputs.stomp.output`). Also, in `debian/control`, the `python3-stomp` package name has been fixed (by removing the `.py` suffix). The changelog has been updated appropriately. lib, bots, doc: STOMP/*n6*-related fixes/enhancements, also ad security SSL-related changes -- regarding `intelmq.lib.mixins.StompMixin` and, therefore, also the *STOMP collector* bot (`StompCollectorBot` defined in `intelmq.bots.collectors.stomp.collector`) and the *STOMP output* bot (`StompOutputBot` defined in `intelmq.bots.outputs.stomp.output`) -- have been made: * *Security*-focused: fixed certain security problems which were caused by the fact that certain versions of the `stomp.py` library we need to be compatible with use the `ssl` module's tools in such ways that suffer from certain *security weaknesses*. In particular, `stomp.py` in versions `>=8.0, <8.1` mistakenly creates an `SSLContext` instance with the `check_hostname` flag unset -- an important negative effect of that is that the hostname of the STOMP server is *not* checked during the TLS handshake (making all STOMP communication vulnerable to certain kinds of attacks...). Also, there are weaknesses (caused either by `stomp.py` or by older, yet still supported by IntelMQ, Python versions) of using too old versions of the TLS protocol (namely: 1.0 and 1.1 -- nowadays considered insecure). * *Admin convenience*-focused: from now on, for each of the STOMP bots, you can set the `ssl_ca_certificate` config param to an empty string -- dictating that the SSL tools employed by the `stomp.py`'s machinery will attempt to load the system’s default CA certificates. Thanks to that, administrators of the given IntelMQ instance can be relieved of of the fuss with manual updates of the CA certificate(s) file -- *if* the certificate of the STOMP server can be verified using some of the publicly available CA certificates which are part of nearly all mainstream operating system distributions (this will be the case with the server certificate of the new variant of the *n6* Stream API, that is, the variant with STOMP-login-and-passcode-based authentication). An important part of the implementation of the aforementioned changes is a non-public class, `intelmq.lib.mixins.stomp._StompPyDedicatedSSLProxy` -- which implements a kind of transparent proxy object that wraps the `ssl` attribute of the `stomp.transport` module (originaly set to the `ssl` module object), replacing some of the `ssl` module's tools with their patched variants (note that the `ssl` module itself and all its members are left untouched). The parts of the IntelMQ's documentation related to those STOMP bots + integration with *n6* (including the CERT.PL's "N6 Stomp Stream" feed description) have been updated and improved; also, the changelog has been updated. bots: fix import logic in STOMP collector's module The logic regarding importing of the `stomp.py`'s stuff has been fixed: now the condition of the absence of the `stomp` module (and thus, of the entire library) would not be confused with the condition of the absence of only the `stomp.exception` module (which would mean the presence of a version of that library lacking just the `exception` submodule). --- CHANGELOG.md | 35 ++- debian/control | 2 +- docs/user/bots.md | 123 ++++++--- docs/user/feeds.md | 16 +- .../bots/collectors/stomp/REQUIREMENTS.txt | 2 +- intelmq/bots/collectors/stomp/collector.py | 44 +++- intelmq/bots/outputs/stomp/REQUIREMENTS.txt | 2 +- intelmq/bots/outputs/stomp/output.py | 41 ++- intelmq/etc/feeds.yaml | 42 ++- intelmq/lib/mixins/stomp.py | 248 +++++++++++++++++- 10 files changed, 446 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b818da034..8423eed38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ if `auth_by_ssl_client_certificate` is *false*); - `password` (STOMP authentication passcode, default: "guest"; to be used only if `auth_by_ssl_client_certificate` is *false*). +- Add the possibility to set the `ssl_ca_certificate` configuration parameter for + `intelmq.bots.collectors.stomp.collector` and/or `intelmq.bots.outputs.stomp.output` + to an empty string - which means that the SSL machinery used for STOMP communication + will attempt to load the system’s default CA certificates (PR#2414 by Jan Kaliszewski). ### Core - `intelmq.lib.message`: For invalid message keys, add a hint on the failure to the exception: not allowed by configuration or not matching regular expression (PR#2398 by Sebastian Wagner). @@ -27,7 +31,7 @@ - `intelmq.lib.mixins`: Add a new class, `StompMixin` (defined in a new submodule: `stomp`), which provides certain common STOMP-bot-specific operations, factored out from `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output` - (PR#2408 by Jan Kaliszewski). + (PR#2408 and PR#2414 by Jan Kaliszewski). ### Development - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner). @@ -36,11 +40,16 @@ ### Bots #### Collectors -- `intelmq.bots.collectors.stomp.collector` (PR#2408 by Jan Kaliszewski): - - Add support for authentication based on STOMP login and passcode, - introducing 3 new configuration parameters (see above: *Configuration*). +- `intelmq.bots.collectors.stomp.collector` (PR#2408 and PR#2414 by Jan Kaliszewski): + - Drop support for versions of `stomp.py` older than `4.1.12`. - Update the code to support new versions of `stomp.py`, including the latest (`8.1.0`); fixes [#2342](https://github.com/certtools/intelmq/issues/2342). + - Add support for authentication based on STOMP login and passcode, introducing three + new configuration parameters (see above: *Configuration*). + - Add support for loading the system’s default CA certificates, as an alternative to + specifying the CA certificate(s) file path explicitly (see above: *Configuration*). + - Fix (by carefully targeted monkey patching) certain security problems caused by + SSL-related weaknesses that some versions of `stomp.py` suffer from. - Fix the reconnection behavior: do not attempt to reconnect after `shutdown`. Also, never attempt to reconnect if the version of `stomp.py` is older than `4.1.21` (it did not work properly anyway). @@ -56,27 +65,35 @@ #### Experts #### Outputs -- `intelmq.bots.outputs.stomp.output` (PR#2408 by Jan Kaliszewski): - - Add support for authentication based on STOMP login and passcode, - introducing 3 new configuration parameters (see above: *Configuration*). +- `intelmq.bots.outputs.stomp.output` (PR#2408 and PR#2414 by Jan Kaliszewski): + - Drop support for versions of `stomp.py` older than `4.1.12`. - Update the code to support new versions of `stomp.py`, including the latest (`8.1.0`). + - Add support for authentication based on STOMP login and passcode, introducing three + new configuration parameters (see above: *Configuration*). + - Add support for loading the system’s default CA certificates, as an alternative to + specifying the CA certificate(s) file path explicitly (see above: *Configuration*). + - Fix (by carefully targeted monkey patching) certain security problems caused by + SSL-related weaknesses that some versions of `stomp.py` suffer from. - Fix `AttributeError` caused by attempts to get unset attributes of `StompOutputBot` (`ssl_ca_cert` et consortes). - Add coercion of the `port` config parameter to `int`. - Add implementation of the `check` hook (verifying, in particular, accessibility of necessary file(s)). - - Add `stomp.py` version check (raise `MissingDependencyError` if not `>=4.1.8`). + - Add `stomp.py` version check (raise `MissingDependencyError` if not `>=4.1.12`). - Minor fixes/improvements and some refactoring (see also above: *Core*...). ### Documentation - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner). - Add a guide of developing extensions packages (PR#2413 by Kamil Mankowski) - Update/fix/improve the stuff related to the STOMP bots and integration with the *n6*'s - Stream API (PR#2408 by Jan Kaliszewski). + Stream API (PR#2408 and PR#2414 by Jan Kaliszewski). - Complete documentation overhaul. Change to markdown format. Uses the mkdocs-material (PR#2419 by Filip Pokorný). ### Packaging - Add `pendulum` to suggested packages, as it is required for the sieve bot (PR#2424 by Sebastian Wagner). +- `debian/control`: in `Suggests` field, replace ``python3-stomp.py (>= 4.1.9)`` with + ``python3-stomp (>= 4.1.12)``, i.e., fix the package name by removing the `.py` + suffix and bump the minimum version to `4.1.12` (PR#2414 by Jan Kaliszewski). ### Tests diff --git a/debian/control b/debian/control index c8f6feef9..b47388c4e 100644 --- a/debian/control +++ b/debian/control @@ -53,7 +53,7 @@ Suggests: python3-geoip2 (>= 2.2.0), python3-pyasn (>= 1.5.0), python3-pymongo (>= 2.7.1), python3-sleekxmpp (>= 1.3.1), - python3-stomp.py (>= 4.1.9), + python3-stomp (>= 4.1.12), python3-pendulum Description: Solution for IT security teams for collecting and processing security feeds IntelMQ is a solution for IT security teams (CERTs, CSIRTs, abuse diff --git a/docs/user/bots.md b/docs/user/bots.md index 29977f56e..2e4e0e059 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -1176,35 +1176,49 @@ Install the `stomp.py` library from PyPI: pip3 install -r intelmq/bots/collectors/stomp/REQUIREMENTS.txt ``` +Alternatively, you may want to install it using your OS's native +packaging tools, e.g.: + +```bash +apt install python3-stomp +``` + +Apart from that, depending on what STOMP server you connect to, you may +need to obtain, from the organization or company owning the server, one +or more of the following security/authentication-related resources: + +* CA certificate file; +* either: *client certificate* and *client certificate's key* files, + or: *username* (STOMP *login*) and *password* (STOMP *passcode*). + +Also, you will need to know an appropriate STOMP *destination* (aka +*exchange point*), e.g. `/exchange/my.example.org/*.*.*.*`. + **Parameters (also expects [feed parameters](#feed-parameters)):** **`server`** -(required, string) Hostname of the STOMP server. +(required, string) STOMP server's hostname or IP, e.g. "n6stream.cert.pl" (which is default) **`port`** -(optional, integer) Defaults to 61614. +(optional, integer) STOMP server's port number (default: 61614) **`exchange`** -(required, string) STOMP *destination* to subscribe to, e.g. "/exchange/my.org/*.*.*.*" +(required, string) STOMP *destination* to subscribe to, e.g. `"/exchange/my.org/*.*.*.*"` -**`username`** - -(optional, string) Username to use. +**`heartbeat`** -**`password`** - -(optional, string) Password to use. +(optional, integer) default: 6000 **`ssl_ca_certificate`** -(optional, string) Path to trusted CA certificate. +(optional, string) Path to CA file, or empty string to load system's default CA certificates **`auth_by_ssl_client_certificate`** -(optional, boolean) Whether to authenticate using TLS certificate. (Set to false for new *n6* auth.) Defaults to true. +(optional, boolean) Default: true (note: false is needed for new *n6* auth) **`ssl_client_certificate`** @@ -1214,6 +1228,14 @@ pip3 install -r intelmq/bots/collectors/stomp/REQUIREMENTS.txt (optional, string) Path to client private key to use for TLS connections. +**`username`** + +(optional, string) Username to use. + +**`password`** + +(optional, string) Password to use. + --- ### Twitter (REMOVE?)
@@ -5127,72 +5149,87 @@ This bot pushes data to any STOMP stream. STOMP stands for Streaming Text Orient **Requirements** -Install the stomp.py library, e.g. [apt install python3-stomp.py] or [pip install stomp.py]. +Install the `stomp.py` library from PyPI: -You need a CA certificate, client certificate and key file from the organization / server you are connecting to. Also -you will need a so called "exchange point". +```bash +pip3 install -r intelmq/bots/outputs/stomp/REQUIREMENTS.txt +``` -**Parameters:** +Alternatively, you may want to install it using your OS's native +packaging tools, e.g.: -**`exchange`** +```bash +apt install python3-stomp +``` -(optional, string) The exchange to push to. Defaults to `/exchange/_push`. +Apart from that, depending on what STOMP server you connect to, you may +need to obtain, from the organization or company owning the server, one +or more of the following security/authentication-related resources: -**`username`** +* CA certificate file; +* either: *client certificate* and *client certificate's key* files, + or: *username* (STOMP *login*) and *password* (STOMP *passcode*). -(optional, string) Username to use. +Also, you will need to know an appropriate STOMP *destination* (aka +*exchange point*), e.g. `/exchange/_push`. -**`password`** +**Parameters:** -(optional, string) Password to use. +**`server`** -**`ssl_ca_certificate`** +(optional, string) STOMP server's hostname or IP, e.g. "n6stream.cert.pl" or "127.0.0.1" (which is default) -(optional, string) Path to trusted CA certificate. +**`port`** -**`auth_by_ssl_client_certificate`** +(optional, integer) STOMP server's port number (default: 61614) -(optional, boolean) Whether to authenticate using TLS certificate. (Set to false for new *n6* auth.) Defaults to true. +**`exchange`** + +(optional, string) STOMP *destination* to push at, e.g. ``"/exchange/_push"`` (which is default) **`heartbeat`** (optional, integer) Defaults to 60000. -**`message_hierarchical_output`** +**`ssl_ca_certificate`** -(optional, boolean) Defaults to false. +(optional, string) path to CA file, or empty string to load system's default CA certificates -**`message_jsondict_as_string`** +**`auth_by_ssl_client_certificate`** -(optional, boolean) Defaults to false. +(optional, boolean) default: true (note: false is needed for new *n6* auth) -**`message_with_type`** +**`ssl_client_certificate`** -(optional, boolean) Defaults to false. +(optional, string) Path to client certificate to use for TLS connections. -**`port`** +**`ssl_client_certificate_key`** -(optional, integer) Defaults to 61614. +(optional, string) Path to client private key to use for TLS connections. -**`server`** +**`username`** -(optional, string) Hostname of the STOMP server. +(optional, string) STOMP *login* (e.g., *n6* user login), used only if `auth_by_ssl_client_certificate` is false -**`single_key`** +**`password`** -(optional, string) Output only a single specified key. In case of `raw` key the data is base64 decoded. Defaults to null (output the whole message). +(optional, string) STOMP *passcode* (e.g., *n6* user API key), used only if `auth_by_ssl_client_certificate` is false -**`ssl_ca_certificate`** +**`message_hierarchical_output`** -(optional, string) Path to trusted CA certificate. +(optional, boolean) Defaults to false. -**`ssl_client_certificate`** +**`message_jsondict_as_string`** -(optional, string) Path to client certificate to use for TLS connections. +(optional, boolean) Defaults to false. -**`ssl_client_certificate_key`** +**`message_with_type`** -(optional, string) Path to client private key to use for TLS connections. +(optional, boolean) Defaults to false. + +**`single_key`** + +(optional, string) Output only a single specified key. In case of `raw` key the data is base64 decoded. Defaults to null (output the whole message). --- diff --git a/docs/user/feeds.md b/docs/user/feeds.md index 1229b3401..8224336d8 100644 --- a/docs/user/feeds.md +++ b/docs/user/feeds.md @@ -719,15 +719,15 @@ parameters: ### N6 Stomp Stream -N6 Collector - CERT.pl's N6 Collector - N6 feed via STOMP interface. Note that rate_limit does not apply for this bot as it is waiting for messages on a stream. +N6 Collector - CERT.pl's *n6* Stream API feed (via STOMP interface). Note that 'rate_limit' does not apply to this bot, as it is waiting for messages on a stream. **Public:** no -**Revision:** 2023-09-23 +**Revision:** 2023-10-08 **Documentation:** -**Additional Information:** Contact cert.pl to get access to the feed. +**Additional Information:** Contact CERT.pl to get access to the feed. Note that the configuration parameter values suggested here are suitable for the new *n6* Stream API variant (with authentication based on 'username' and 'password'); for this variant, typically you can leave the 'ssl_ca_certificate' parameter's value empty - then the system's default CA certificates will be used; however, if that does not work, you need to set 'ssl_ca_certificate' to the path to a file containing CA certificates eligible to verify "*.cert.pl" server certificates (to be found among the publicly available CA certs distributed with modern web browsers/OSes). Also, note that the 'server' parameter's value (for the *new API variant*) suggested here, "n6stream-new.cert.pl", is a temporary domain; ultimately, it will be changed back to "stream.cert.pl". When it comes to the *old API variant* (turned off in November 2023!), you need to have the 'server' parameter set to the name "n6stream.cert.pl", 'auth_by_ssl_client_certificate' set to true, 'ssl_ca_certificate' set to the path to a file containing the *n6*'s legacy self-signed CA certificate (which is stored in file "intelmq/bots/collectors/stomp/ca.pem"), and the parameters 'ssl_client_certificate' and 'ssl_client_certificate_key' set to the paths to your-*n6*-client-specific certificate and key files (note that the 'username' and 'password' parameters are then irrelevant and can be omitted). **Collector configuration** @@ -736,14 +736,14 @@ N6 Collector - CERT.pl's N6 Collector - N6 feed via STOMP interface. Note that r module: intelmq.bots.collectors.stomp.collector parameters: auth_by_ssl_client_certificate: False - exchange: {insert your exchange point as given by CERT.pl} + exchange: {insert your STOMP *destination* to subscribe to, as given by CERT.pl, e.g. /exchange/my.example.org/*.*.*.*} name: N6 Stomp Stream - password: {insert n6 user's API key} + password: {insert your *n6* API key} port: 61614 provider: CERT.PL - server: n6stream.cert.pl - ssl_ca_certificate: {insert path to CA file for CERT.pl's n6} - username: {insert n6 user's login} + server: n6stream-new.cert.pl + ssl_ca_certificate: + username: {insert your *n6* login, e.g. someuser@my.example.org} ``` **Parser configuration** diff --git a/intelmq/bots/collectors/stomp/REQUIREMENTS.txt b/intelmq/bots/collectors/stomp/REQUIREMENTS.txt index 08a80aca8..de080c955 100644 --- a/intelmq/bots/collectors/stomp/REQUIREMENTS.txt +++ b/intelmq/bots/collectors/stomp/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2017 Sebastian Wagner # SPDX-License-Identifier: AGPL-3.0-or-later -stomp.py>=4.1.8 +stomp.py>=4.1.12 diff --git a/intelmq/bots/collectors/stomp/collector.py b/intelmq/bots/collectors/stomp/collector.py index 86ebee40c..28ff63e22 100644 --- a/intelmq/bots/collectors/stomp/collector.py +++ b/intelmq/bots/collectors/stomp/collector.py @@ -4,15 +4,19 @@ # -*- coding: utf-8 -*- -from intelmq.lib.bot import CollectorBot -from intelmq.lib.mixins import StompMixin - try: import stomp - import stomp.exception except ImportError: stomp = None else: + import stomp.exception + +from intelmq.lib.bot import CollectorBot +from intelmq.lib.mixins import StompMixin + + +if stomp is not None: + class StompListener(stomp.PrintingListener): """ the stomp listener gets called asynchronously for @@ -74,17 +78,33 @@ def connect_and_subscribe(conn, logger, destination, start=False, connect_kwargs class StompCollectorBot(CollectorBot, StompMixin): """Collect data from a STOMP Interface""" """ main class for the STOMP protocol collector """ - exchange: str = '' + + server: str = 'n6stream.cert.pl' port: int = 61614 - server: str = "n6stream.cert.pl" - auth_by_ssl_client_certificate: bool = True - username: str = 'guest' # ignored if `auth_by_ssl_client_certificate` is true - password: str = 'guest' # ignored if `auth_by_ssl_client_certificate` is true - ssl_ca_certificate: str = 'ca.pem' # TODO pathlib.Path - ssl_client_certificate: str = 'client.pem' # TODO pathlib.Path - ssl_client_certificate_key: str = 'client.key' # TODO pathlib.Path + exchange: str = '' heartbeat: int = 6000 + # Note: the `ssl_ca_certificate` configuration parameter must be set: + # * *either* to the server's CA certificate(s) file path, + # * *or* to an empty string -- dictating that the SSL tools employed + # by the `stomp.py`'s machinery will attempt to load the system’s + # default CA certificates. + # The latter, if applicable, is more convenient -- by avoiding the + # need to manually update the CA certificate(s) file. + ssl_ca_certificate: str = 'ca.pem' # <- TODO: change to '' (+ remove "ca.pem*" legacy files) + # (^ TODO: could also be pathlib.Path) + + auth_by_ssl_client_certificate: bool = True + + # Used if `auth_by_ssl_client_certificate` is true (otherwise ignored): + ssl_client_certificate: str = 'client.pem' # (cert file path) + ssl_client_certificate_key: str = 'client.key' # (cert's key file path) + # (^ TODO: could also be pathlib.Path) + + # Used if `auth_by_ssl_client_certificate` is false (otherwise ignored): + username: str = 'guest' # (STOMP auth *login*) + password: str = 'guest' # (STOMP auth *passcode*) + _collector_empty_process: bool = True __conn = False # define here so shutdown method can check for it diff --git a/intelmq/bots/outputs/stomp/REQUIREMENTS.txt b/intelmq/bots/outputs/stomp/REQUIREMENTS.txt index 24956e994..0b8717875 100644 --- a/intelmq/bots/outputs/stomp/REQUIREMENTS.txt +++ b/intelmq/bots/outputs/stomp/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2016 aaronkaplan # SPDX-License-Identifier: AGPL-3.0-or-later -stomp.py>=4.1.8 +stomp.py>=4.1.12 diff --git a/intelmq/bots/outputs/stomp/output.py b/intelmq/bots/outputs/stomp/output.py index a28de3f4e..50c9a1d5f 100644 --- a/intelmq/bots/outputs/stomp/output.py +++ b/intelmq/bots/outputs/stomp/output.py @@ -4,34 +4,51 @@ # -*- coding: utf-8 -*- -from intelmq.lib.bot import OutputBot -from intelmq.lib.mixins import StompMixin - try: import stomp except ImportError: stomp = None +from intelmq.lib.bot import OutputBot +from intelmq.lib.mixins import StompMixin + class StompOutputBot(OutputBot, StompMixin): """Send events to a STMOP server""" """ main class for the STOMP protocol output bot """ - exchange: str = "/exchange/_push" - heartbeat: int = 60000 + http_verify_cert = True keep_raw_field: bool = False message_hierarchical_output: bool = False message_jsondict_as_string: bool = False message_with_type: bool = False - port: int = 61614 - server: str = "127.0.0.1" # TODO: could be ip address single_key: bool = False + + server: str = '127.0.0.1' # <- TODO: change to 'n6stream.cert.pl' (==StompCollectorBot.server) + port: int = 61614 + exchange: str = '/exchange/_push' + heartbeat: int = 60000 + + # Note: the `ssl_ca_certificate` configuration parameter must be set: + # * *either* to the server's CA certificate(s) file path, + # * *or* to an empty string -- dictating that the SSL tools employed + # by the `stomp.py`'s machinery will attempt to load the system’s + # default CA certificates. + # The latter, if applicable, is more convenient -- by avoiding the + # need to manually update the CA certificate(s) file. + ssl_ca_certificate: str = 'ca.pem' # <- TODO: change to '' + # (^ TODO: could also be pathlib.Path) + auth_by_ssl_client_certificate: bool = True - username: str = 'guest' # ignored if `auth_by_ssl_client_certificate` is true - password: str = 'guest' # ignored if `auth_by_ssl_client_certificate` is true - ssl_ca_certificate: str = 'ca.pem' # TODO: could be pathlib.Path - ssl_client_certificate: str = 'client.pem' # TODO: pathlib.Path - ssl_client_certificate_key: str = 'client.key' # TODO: patlib.Path + + # Used if `auth_by_ssl_client_certificate` is true (otherwise ignored): + ssl_client_certificate: str = 'client.pem' # (cert file path) + ssl_client_certificate_key: str = 'client.key' # (cert's key file path) + # (^ TODO: could also be pathlib.Path) + + # Used if `auth_by_ssl_client_certificate` is false (otherwise ignored): + username: str = 'guest' # (STOMP auth *login*) + password: str = 'guest' # (STOMP auth *passcode*) _conn = None diff --git a/intelmq/etc/feeds.yaml b/intelmq/etc/feeds.yaml index 3a509e35c..f87c9509f 100644 --- a/intelmq/etc/feeds.yaml +++ b/intelmq/etc/feeds.yaml @@ -1149,27 +1149,49 @@ providers: public: false CERT.PL: N6 Stomp Stream: - description: N6 Collector - CERT.pl's N6 Collector - N6 feed via STOMP interface. - Note that rate_limit does not apply for this bot as it is waiting for messages + description: N6 Collector - CERT.pl's *n6* Stream API feed (via STOMP interface). + Note that 'rate_limit' does not apply to this bot, as it is waiting for messages on a stream. - additional_information: Contact cert.pl to get access to the feed. + additional_information: Contact CERT.pl to get access to the feed. + Note that the configuration parameter values suggested here are + suitable for the new *n6* Stream API variant (with authentication + based on 'username' and 'password'); for this variant, typically + you can leave the 'ssl_ca_certificate' parameter's value empty - + then the system's default CA certificates will be used; however, + if that does not work, you need to set 'ssl_ca_certificate' to + the path to a file containing CA certificates eligible to verify + "*.cert.pl" server certificates (to be found among the publicly + available CA certs distributed with modern web browsers/OSes). + Also, note that the 'server' parameter's value (for the *new API + variant*) suggested here, "n6stream-new.cert.pl", is a temporary + domain; ultimately, it will be changed back to "stream.cert.pl". + When it comes to the *old API variant* (turned off in November + 2023!), you need to have the 'server' parameter set to the name + "n6stream.cert.pl", 'auth_by_ssl_client_certificate' set to + true, 'ssl_ca_certificate' set to the path to a file containing + the *n6*'s legacy self-signed CA certificate (which is stored in + file "intelmq/bots/collectors/stomp/ca.pem"), and the parameters + 'ssl_client_certificate' and 'ssl_client_certificate_key' set to + the paths to your-*n6*-client-specific certificate and key files + (note that the 'username' and 'password' parameters are then + irrelevant and can be omitted). bots: collector: module: intelmq.bots.collectors.stomp.collector parameters: - exchange: "{insert your exchange point as given by CERT.pl}" - ssl_ca_certificate: "{insert path to CA file for CERT.pl's n6}" + exchange: "{insert your STOMP *destination* to subscribe to, as given by CERT.pl, e.g. /exchange/my.example.org/*.*.*.*}" + server: "n6stream-new.cert.pl" + port: 61614 + ssl_ca_certificate: "" auth_by_ssl_client_certificate: false - username: "{insert n6 user's login}" - password: "{insert n6 user's API key}" - port: '61614' - server: n6stream.cert.pl + username: "{insert your *n6* login, e.g. someuser@my.example.org}" + password: "{insert your *n6* API key}" name: __FEED__ provider: __PROVIDER__ parser: module: intelmq.bots.parsers.n6.parser_n6stomp parameters: - revision: 2023-09-23 + revision: 2023-10-08 documentation: https://n6.readthedocs.io/usage/streamapi/ public: false AlienVault: diff --git a/intelmq/lib/mixins/stomp.py b/intelmq/lib/mixins/stomp.py index 41cbd29cb..9ef966e66 100644 --- a/intelmq/lib/mixins/stomp.py +++ b/intelmq/lib/mixins/stomp.py @@ -4,18 +4,25 @@ SPDX-License-Identifier: AGPL-3.0-or-later """ +import enum +import os +import ssl +import sys from typing import ( Any, Callable, List, NoReturn, Tuple, + Union, ) try: import stomp except ImportError: stomp = None +else: + import stomp.transport from intelmq.lib.exceptions import MissingDependencyError @@ -31,14 +38,26 @@ class StompMixin: port: int heartbeat: int + # Note: the `ssl_ca_certificate` configuration parameter must be set: + # * *either* to the server's CA certificate(s) file path, + # * *or* to an empty string -- dictating that the SSL tools employed + # by the `stomp.py`'s machinery will attempt to load the system’s + # default CA certificates. + # The latter, if applicable, is more convenient -- by avoiding the + # need to manually update the CA certificate(s) file. + ssl_ca_certificate: str + # (^ TODO: could also be pathlib.Path) + auth_by_ssl_client_certificate: bool - username: str # to be ignored if `auth_by_ssl_client_certificate` is true - password: str # to be ignored if `auth_by_ssl_client_certificate` is true + # Used if `auth_by_ssl_client_certificate` is true (otherwise ignored): + ssl_client_certificate: str # (cert file path) + ssl_client_certificate_key: str # (cert's key file path) + # (^ TODO: could also be pathlib.Path) - ssl_ca_certificate: str # TODO: could be pathlib.Path - ssl_client_certificate: str # TODO: could be pathlib.Path - ssl_client_certificate_key: str # TODO: could be patlib.Path + # Used if `auth_by_ssl_client_certificate` is false (otherwise ignored): + username: str # (STOMP auth *login*) + password: str # (STOMP auth *passcode*) # # Helper methods intended to be used in subclasses @@ -73,7 +92,11 @@ def prepare_stomp_connection(self) -> Tuple['stomp.Connection', dict]: to be passed to the `connect()` method of the aforementioned `` object. """ + _StompPyDedicatedSSLProxy.patch_stomp_transport_ssl() ssl_kwargs, connect_kwargs = self.__get_ssl_and_connect_kwargs() + # Note: here we coerce `port` to int just to be on the safe + # side, as some historical versions of `etc/feeds.yaml` used + # to set it to a string. host_and_ports = [(self.server, int(self.port))] stomp_connection = stomp.Connection(host_and_ports=host_and_ports, heartbeats=(self.heartbeat, @@ -98,8 +121,8 @@ def __verify_dependency(cls) -> None: if stomp is None: raise MissingDependencyError('stomp', additional_text=cls._DEPENDENCY_NAME_REMARK) - if stomp.__version__ < (4, 1, 8): - raise MissingDependencyError('stomp', version="4.1.8", + if stomp.__version__ < (4, 1, 12): + raise MissingDependencyError('stomp', version="4.1.12", installed=stomp.__version__, additional_text=cls._DEPENDENCY_NAME_REMARK) @@ -107,7 +130,9 @@ def __verify_dependency(cls) -> None: def __verify_parameters(cls, get_param: Callable[[str], Any], on_error: Callable[[str], None]) -> None: - file_param_names = ['ssl_ca_certificate'] + file_param_names = [] + if get_param('ssl_ca_certificate'): + file_param_names.append('ssl_ca_certificate') if cls.__should_cert_auth_params_be_verified(get_param, on_error): file_param_names.extend([ 'ssl_client_certificate', @@ -154,10 +179,12 @@ def __raise_value_error(self, msg: str) -> NoReturn: raise ValueError(msg) def __get_ssl_and_connect_kwargs(self) -> Tuple[dict, dict]: - # Note: the `ca_certs` argument to `set_ssl()` must always be - # provided, otherwise the `stomp.py`'s machinery would *not* - # perform any certificate verification! - ssl_kwargs = dict(ca_certs=self.ssl_ca_certificate) + # Note: a *non-empty* and *non-None* `ca_certs` argument must + # always be passed to `set_ssl()`; otherwise the `stomp.py`'s + # machinery would *not* enable any certificate verification! + ssl_kwargs = dict(ca_certs=( + self.ssl_ca_certificate if self.ssl_ca_certificate + else _SYSTEM_DEFAULT_CA_MARKER)) connect_kwargs = dict(wait=True) if self.auth_by_ssl_client_certificate: ssl_kwargs.update( @@ -170,3 +197,200 @@ def __get_ssl_and_connect_kwargs(self) -> Tuple[dict, dict]: passcode=self.password, ) return ssl_kwargs, connect_kwargs + + +# Note: internally, we need to use a non-empty marker string because the +# logic of the `stomp.py`'s machinery does not make it possible to use +# None or an empty string as a request to load the system's default CA +# certificates. Also, note that the string is intentionally an absolute +# filesystem path which *obviously does not point to an existing file* +# -- in case the value was used, by accident, as a CA certificate file +# path (as it is better to crash than to allow for silent misbehavior). +_SYSTEM_DEFAULT_CA_MARKER = '/SYSTEM-DEFAULT-CA-SPECIAL-INTELMQ-MARKER/' + + +class _StompPyDedicatedSSLProxy: + + """ + A kind of proxy to wrap the `stomp.transport` module's `ssl` member + (originally being an object representing the standard `ssl` module), + replacing some `ssl`-provided tools with their patched variants. + + We need it to fix the following two problems: + + * (1) Certain versions of `stomp.py` we need to be compatible with + use the `ssl` module's tools in such ways that suffer from certain + *security weaknesses*. (In particular, `stomp.py >=8.0, <8.1` + creates an `SSLContext` instance with the `check_hostname` flag + unset -- an important negative effect is that the hostname of the + STOMP server is *not* checked during the TLS handshake! See also + code comments...) + + * (2) No version of `stomp.py` (at least as of this writing, i.e., up + to and including `8.1.0`) makes it possible to load the *system's + default CA certificates* -- condemning us to bother with manual + updates of the CA certificate(s) file, even if the certificate of + the STOMP server we connect to could be verified using some of the + publicly available CA certificates which are part of nearly all + mainstream operating system distributions (this is the case with + the new *n6* Stream API server's certificate). + + Note that the `ssl` module itself and all its members (as seen from + anywhere else than the `stomp.transport` module) are left untouched. + Just the `ssl` member of the `stomp.transport` module is replaced + with an instance of this class (it is done by invoking the class + method `_StompPyDedicatedSSLProxy.patch_stomp_transport_ssl()`). + + *** + + The implementation of this class assumes that: + + * the Python version is `>= 3.7` (guaranteed thanks to the IntelMQ's + project/setup declarations); + * the `stomp.py` dependency is installed and its version is always + `>= 4.1.12` (guaranteed thanks to STOMP bots' `REQUIREMENTS.txt`; + see also: the `StompMixin.__verify_dependency()` method invoked in + the `StompMixin.stomp_bot_runtime_initial_check()` method); + * the `stomp` importable module has the `transport` submodule (see + the `import stomp.transport` near the beginning of the source code + of the module in which `_StompPyDedicatedSSLProxy` is defined). + """ + + # + # Checking and replacing `stomp.transport` module's `ssl` member + + @classmethod + def patch_stomp_transport_ssl(cls) -> None: + if getattr(stomp.transport, 'DEFAULT_SSL_VERSION', None) is None: + raise NotImplementedError('stomp.transport.DEFAULT_SSL_VERSION' + 'not found or None') + found_ssl = getattr(stomp.transport, 'ssl', None) + if found_ssl is ssl: + # (patch only if not already patched!) + stomp.transport.ssl = cls() + elif not isinstance(found_ssl, cls): + raise NotImplementedError(f'unexpectedly, stomp.transport.ssl ' + f'is neither {ssl!r} nor an instance ' + f'of {cls!r} (found: {found_ssl!r})') + + # + # Proxying/substituting `ssl` tools for `stomp.transport` module + + def __dir__(self) -> List[str]: + return dir(ssl) + + def __getattribute__(self, name: str) -> Any: + # Selected `ssl` module's members are replaced with their patched + # variants (see their definitions below...). + if name in {'SSLContext', 'create_default_context'}: + return super().__getattribute__(name) + + # The rest of the `ssl` module's members are just retrieved from + # that module: + return getattr(ssl, name) + + def __setattr__(self, name: str, value: Any) -> None: + raise NotImplementedError('setting attributes on stomp.' + 'transport.ssl is not supported') + + def __delattr__(self, name: str) -> None: + raise NotImplementedError('deleting attributes from stomp.' + 'transport.ssl is not supported') + + class SSLContext(ssl.SSLContext): + + """ + Note: `ssl.SSLContext` is invoked directly by `stomp.py >= 8.0.0`. + Here we subclass it to handle our `_SYSTEM_DEFAULT_CA_MARKER` as + well as to ensure that certain important security-related stuff is + in accordance with the Python core developers' recommendations (see: + https://docs.python.org/library/ssl.html#security-considerations) + and that the TLS version we use is not too old... + """ + + def __new__(cls, + protocol: Union[int, enum.Enum, None] = None, + *args, + **kwargs) -> '_StompPyDedicatedSSLProxy.SSLContext': + # Note: the `stomp.py`'s machinery *ignores* `ssl_version` + # got by `stomp.Connection.set_ssl()`, and passes to the + # `ssl.SSLContext` constructor the value of the constant + # `stomp.transport.DEFAULT_SSL_VERSION`. However, because + # `PROTOCOL_TLS_CLIENT` is a good modern setting, we use it + # instead of `stomp.transport.DEFAULT_SSL_VERSION` (which, + # if not already set to `PROTOCOL_TLS_CLIENT`, must have + # been set to some older setting -- depending on the version + # of `stomp.py`...). + ssl_context = super().__new__( + cls, + ssl.PROTOCOL_TLS_CLIENT, + *args, + **kwargs) + # The versions of Python older than 3.10 seem to refrain + # from blocking the use of the TLS versions 1.0 and 1.1 + # which nowadays are considered insecure. Let's fix that: + ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 + return ssl_context + + def load_verify_locations(self, + cafile: Union[str, None] = None, + capath: Union[str, None] = None, + cadata: Union[str, bytes, None] = None) -> None: + if cafile == _SYSTEM_DEFAULT_CA_MARKER and not (capath or cadata): + self.load_default_certs(ssl.Purpose.SERVER_AUTH) + else: + super().load_verify_locations(cafile, capath, cadata) + + def wrap_socket(self, + *args, + **kwargs) -> ssl.SSLSocket: + # Let's be sure that nothing spoiled these two SSL context's + # settings, as they are crucial for certificate verification! + if self.verify_mode != ssl.CERT_REQUIRED: + raise ValueError(f"value of SSL context's `verify_mode` " + f"setting ({self.verify_mode!r}) is, " + f"unexpectedly, different from " + f"{ssl.CERT_REQUIRED!r}") + if not self.check_hostname: + raise ValueError(f"value of SSL context's `check_hostname` " + f"setting ({self.check_hostname!r}) is, " + f"unexpectedly, not true") + return super().wrap_socket(*args, **kwargs) + + @classmethod + def create_default_context(cls, + purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH, + *, + cafile: Union[str, None] = None, + capath: Union[str, None] = None, + cadata: Union[str, bytes, None] = None) -> ssl.SSLContext: + + """ + Note: the `ssl.create_default_context()` helper is used by + `stomp.py >= 4.1.12, < 8.0.0`. That is OK, except that we + also want to handle our `_SYSTEM_DEFAULT_CA_MARKER` as well + as to provide some additional security-related tweaks and + checks -- provided by our custom subclass of `SSLContext`. + """ + + if purpose == ssl.Purpose.SERVER_AUTH: + ssl_context = cls.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if not (cafile or capath or cadata): + cafile = _SYSTEM_DEFAULT_CA_MARKER + ssl_context.load_verify_locations(cafile, capath, cadata) + + if sys.version_info[:2] >= (3, 8): + # Support for OpenSSL 1.1.1 keylog (copied from `Py>=3.8`): + if hasattr(ssl_context, 'keylog_filename'): + keylogfile = os.environ.get('SSLKEYLOGFILE') + if keylogfile and not sys.flags.ignore_environment: + ssl_context.keylog_filename = keylogfile + + else: + ssl_context = ssl.create_default_context( + purpose, + cafile=cafile, + capath=capath, + cadata=cadata) + + return ssl_context