From 4da7f62ce354afe543ea60815c75f7da0328e0a4 Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Fri, 10 Mar 2023 17:53:52 +0100 Subject: [PATCH] Standardize and add SSL settings (#168) This commit made the plugin SSL settings consistent with the naming convention defined in the meta issue: https://github.com/elastic/logstash/issues/14905. It added the following SSL settings: ssl_enabled: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts' scheme ssl_certificate: OpenSSL-style X.509 certificate file to authenticate the client ssl_key: OpenSSL-style RSA private key that corresponds to the ssl_certificate ssl_truststore_path: The JKS truststore to validate the server's certificate ssl_truststore_type: The format of the truststore file ssl_truststore_password: The truststore password ssl_keystore_path: The keystore used to present a certificate to the server ssl_keystore_type: The format of the keystore file ssl_keystore_password: The keystore password ssl_cipher_suites: The list of cipher suites to use ssl_supported_protocols: Supported protocols with versions ssl_verification_mode: Defines how to verify the certificates presented by another party in the TLS connection And deprecated: ssl in favor of ssl_enabled ca_file in favor of ssl_certificate_authorities keystore in favor of ssl_keystore_path keystore_password in favor of ssl_keystore_password --- .ci/logstash-run.sh | 11 +- CHANGELOG.md | 20 ++ docs/index.asciidoc | 214 ++++++++++++-- lib/logstash/filters/elasticsearch.rb | 185 ++++++++++-- lib/logstash/filters/elasticsearch/client.rb | 21 +- logstash-filter-elasticsearch.gemspec | 4 +- spec/filters/elasticsearch_spec.rb | 9 +- spec/filters/elasticsearch_ssl_spec.rb | 264 ++++++++++++++++++ .../filters/integration/elasticsearch_spec.rb | 12 +- 9 files changed, 669 insertions(+), 71 deletions(-) create mode 100644 spec/filters/elasticsearch_ssl_spec.rb diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index 0a71208..5ce754f 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -24,9 +24,14 @@ wait_for_es() { } if [[ "$INTEGRATION" != "true" ]]; then - jruby -rbundler/setup -S rspec -fd -t ~integration spec/filters + bundle exec rspec --format=documentation spec/filters --tag ~integration --tag ~secure_integration else - extra_tag_args="-t integration" + if [[ "$SECURE_INTEGRATION" == "true" ]]; then + extra_tag_args="--tag secure_integration" + else + extra_tag_args="--tag ~secure_integration --tag integration" + fi + wait_for_es - jruby -rbundler/setup -S rspec -fd $extra_tag_args -t es_version:$ELASTIC_STACK_VERSION spec/filters/integration + bundle exec rspec --format=documentation $extra_tag_args --tag update_tests:painless --tag es_version:$ELASTIC_STACK_VERSION spec/filters/integration fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 7774f4d..c1d7ad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,23 @@ +## 3.15.0 + - Added SSL settings for: [#168](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/168) + - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme + - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client + - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate` + - `ssl_truststore_path`: The JKS truststore to validate the server's certificate + - `ssl_truststore_type`: The format of the truststore file + - `ssl_truststore_password`: The truststore password + - `ssl_keystore_path`: The keystore used to present a certificate to the server + - `ssl_keystore_type`: The format of the keystore file + - `ssl_keystore_password`: The keystore password + - `ssl_cipher_suites`: The list of cipher suites to use + - `ssl_supported_protocols`: Supported protocols with versions + - `ssl_verification_mode`: Defines how to verify the certificates presented by another party in the TLS connection + - Reviewed and deprecated SSL settings to comply with Logstash's naming convention + - Deprecated `ssl` in favor of `ssl_enabled` + - Deprecated `ca_file` in favor of `ssl_certificate_authorities` + - Deprecated `keystore` in favor of `ssl_keystore_path` + - Deprecated `keystore_password` in favor of `ssl_keystore_password` + ## 3.14.0 - Added support for configurable retries with new `retry_on_failure` and `retry_on_status` options [#160](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/160) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 8150034..a433757 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -121,14 +121,13 @@ The `monitoring` permission at cluster level is necessary to perform periodic co [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Filter Configuration Options -This plugin supports the following configuration options plus the <> described later. +This plugin supports the following configuration options plus the <> and the <> described later. [cols="<,<,<",options="header",] |======================================================================= |Setting |Input type|Required | <> |<>|No | <> |<>|No -| <> |a valid filesystem path|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -143,11 +142,22 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No -| <> |<>|No +| <> |<>|No | <> |<>|No -| <> |<>|No -| <> |a valid filesystem path|No -| <> |<>|No +| <> |<>|__Deprecated__ +| <> |<>|No +| <> |list of <>|No +| <> |list of <>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["full", "none"]`|No | <> |<>|No | <> |<>|No |======================================================================= @@ -182,19 +192,11 @@ Example: * There is no default value for this setting. Authenticate using Elasticsearch API key. Note that this option also requires -enabling the `ssl` option. +enabling the <> option. Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. -[id="plugins-{type}s-{plugin}-ca_file"] -===== `ca_file` - - * Value type is <> - * There is no default value for this setting. - -SSL Certificate Authority file - [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"] ===== `ca_trusted_fingerprint` @@ -364,30 +366,140 @@ Which HTTP Status codes to consider for retries (in addition to connection error Comma-delimited list of `:` pairs that define the sort order -[id="plugins-{type}s-{plugin}-ssl"] -===== `ssl` +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + * Value type is <> + * There is no default value for this setting. - * Value type is <> - * Default value is `false` +SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file. -SSL +NOTE: This setting can be used only if <> is set. -[id="plugins-{type}s-{plugin}-keystore"] -===== `keystore` +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + + * Value type is a list of <> + * There is no default value for this setting + +The .cer or .pem files to validate the server's certificate. + +NOTE: You cannot use this setting and <> at the same time. + +[id="plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + * Value type is a list of <> + * There is no default value for this setting + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on the Java and protocol versions. + + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + + * Value type is <> + * There is no default value for this setting. + +Enable SSL/TLS secured communication to Elasticsearch cluster. +Leaving this unspecified will use whatever scheme is specified in the URLs listed in <> or extracted from the <>. +If no explicit protocol is specified plain HTTP will be used. +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` * Value type is <> * There is no default value for this setting. -The keystore used to present a certificate to the server. It can be either .jks or .p12 +OpenSSL-style RSA private key that corresponds to the <>. -[id="plugins-{type}s-{plugin}-keystore_password"] -===== `keystore_password` +NOTE: This setting can be used only if <> is set. + +[id="plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` * Value type is <> * There is no default value for this setting. Set the keystore password +[id="plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + + * Value type is <> + * There is no default value for this setting. + +The keystore used to present a certificate to the server. +It can be either `.jks` or `.p12` + +NOTE: You cannot use this setting and <> at the same time. + +[id="plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the keystore filename. + +The format of the keystore file. It must be either `jks` or `pkcs12`. + +[id="plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is <> + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + +[id="plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + + * Value type is <> + * There is no default value for this setting. + +Set the truststore password + +[id="plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + + * Value type is <> + * There is no default value for this setting. + +The truststore to validate the server's certificate. +It can be either `.jks` or `.p12`. + +NOTE: You cannot use this setting and <> at the same time. + +[id="plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + + * Value can be any of: `jks`, `pkcs12` + * If not provided, the value will be inferred from the truststore filename. + +The format of the truststore file. It must be either `jks` or `pkcs12`. + +[id="plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + + * Value can be any of: `full`, `none` + * Default value is `full` + +Defines how to verify the certificates presented by another party in the TLS connection: + +`full` validates that the server certificate has an issue date that’s within +the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and +has a hostname or IP address that matches the names within the certificate. + +`none` performs no certificate validation. + +WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf + [id="plugins-{type}s-{plugin}-tag_on_failure"] ===== `tag_on_failure` @@ -405,5 +517,57 @@ Tags the event on failure to look up previous log event information. This can be Basic Auth - username +[id="plugins-{type}s-{plugin}-deprecated-options"] +==== Elasticsearch Filter Deprecated Configuration Options + +This plugin supports the following deprecated configurations. + +WARNING: Deprecated options are subject to removal in future releases. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting|Input type|Replaced by +| <> |a valid filesystem path|<> +| <> |a valid filesystem path|<> +| <> |<>|<> +|======================================================================= + +[id="plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` +deprecated[3.15.0, Replaced by <>] + +* Value type is <> +* There is no default value for this setting. + +SSL Certificate Authority file + +[id="plugins-{type}s-{plugin}-ssl"] +===== `ssl` +deprecated[3.15.0, Replaced by <>] + +* Value type is <> +* Default value is `false` + +SSL + +[id="plugins-{type}s-{plugin}-keystore"] +===== `keystore` +deprecated[3.15.0, Replaced by <>] + +* Value type is <> +* There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` +deprecated[3.15.0, Replaced by <>] + +* Value type is <> +* There is no default value for this setting. + +Set the keystore password + + [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index 38a9858..03f935b 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -3,6 +3,7 @@ require "logstash/namespace" require "logstash/json" require 'logstash/plugin_mixins/ca_trusted_fingerprint_support' +require "logstash/plugin_mixins/normalize_config_support" require_relative "elasticsearch/client" require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore" @@ -61,17 +62,62 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base config :proxy, :validate => :uri_or_empty # SSL - config :ssl, :validate => :boolean, :default => false + config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead." # SSL Certificate Authority file - config :ca_file, :validate => :path + config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." # The keystore used to present a certificate to the server. # It can be either .jks or .p12 - config :keystore, :validate => :path + config :keystore, :validate => :path, :deprecated => "Use 'ssl_keystore_path' instead." # Set the keystore password - config :keystore_password, :validate => :password + config :keystore_password, :validate => :password, :deprecated => "Use 'ssl_keystore_password' instead." + + # OpenSSL-style X.509 certificate certificate to authenticate the client + config :ssl_certificate, :validate => :path + + # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary + config :ssl_certificate_authorities, :validate => :path, :list => true + + # The list of cipher suites to use, listed by priorities. + # Supported cipher suites vary depending on which version of Java is used. + config :ssl_cipher_suites, :validate => :string, :list => true + + # SSL + config :ssl_enabled, :validate => :boolean + + # OpenSSL-style RSA private key to authenticate the client + config :ssl_key, :validate => :path + + # Set the keystore password + config :ssl_keystore_password, :validate => :password + + # The keystore used to present a certificate to the server. + # It can be either .jks or .p12 + config :ssl_keystore_path, :validate => :path + + # The format of the keystore file. It must be either jks or pkcs12 + config :ssl_keystore_type, :validate => %w[pkcs12 jks] + + # Supported protocols with versions. + config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true + + # Set the truststore password + config :ssl_truststore_password, :validate => :password + + # The JKS truststore to validate the server's certificate. + # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities` + config :ssl_truststore_path, :validate => :path + + # The format of the truststore file. It must be either jks or pkcs12 + config :ssl_truststore_type, :validate => %w[pkcs12 jks] + + # Options to verify the server's certificate. + # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; + # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. + # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + config :ssl_verification_mode, :validate => %w[full none], :default => 'full' # Whether results should be sorted or not config :enable_sort, :validate => :boolean, :default => true @@ -91,6 +137,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # config :ca_trusted_fingerprint, :validate => :sha_256_hex include LogStash::PluginMixins::CATrustedFingerprintSupport + include LogStash::PluginMixins::NormalizeConfigSupport + attr_reader :clients_pool ## @@ -122,13 +170,10 @@ def register @query_dsl = file.read end - if @keystore_password && !@keystore - fail "`keystore_password` was provided, without a `keystore`" - end - + fill_hosts_from_cloud_id + setup_ssl_params! validate_authentication fill_user_password_from_cloud_auth - fill_hosts_from_cloud_id @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s @@ -219,16 +264,80 @@ def client_options :password => @password, :api_key => @api_key, :proxy => @proxy, - :ssl => @ssl, - :ca_file => @ca_file, + :ssl => client_ssl_options, :retry_on_failure => @retry_on_failure, - :retry_on_status => @retry_on_status, - :keystore => @keystore, - :keystore_password => @keystore_password, - :ssl_trust_strategy => trust_strategy_for_ca_trusted_fingerprint + :retry_on_status => @retry_on_status } end + def client_ssl_options + ssl_options = {} + ssl_options[:enabled] = @ssl_enabled + + # If the deprecated `ssl` option was explicitly provided, it keeps the same behavior + # setting up all the client SSL configs even if ssl => false. Otherwise, it should use + # the @ssl_enabled value as it was either explicitly set by the `ssl_enabled` option or + # inferred from the hosts scheme. + return ssl_options unless @ssl_enabled || original_params.include?('ssl') + + ssl_options[:enabled] = true + ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path') + + if ssl_certificate_authorities && ssl_truststore_path + raise LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate' + end + + if ssl_certificate && ssl_keystore_path + raise LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates' + end + + if ssl_certificate_authorities&.any? + raise LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1 + ssl_options[:ca_file] = ssl_certificate_authorities.first + end + + setup_client_ssl_store(ssl_options, 'truststore', ssl_truststore_path) + setup_client_ssl_store(ssl_options, 'keystore', ssl_keystore_path) + logger.debug("Keystore for client certificate", :keystore => ssl_keystore_path) if ssl_keystore_path + + ssl_key = params["ssl_key"] + if ssl_certificate + raise LogStash::ConfigurationError, 'Using an "ssl_certificate" requires an "ssl_key"' unless ssl_key + ssl_options[:client_cert] = ssl_certificate + ssl_options[:client_key] = ssl_key + elsif !ssl_key.nil? + raise LogStash::ConfigurationError, 'An "ssl_certificate" is required when using an "ssl_key"' + end + + ssl_verification_mode = params["ssl_verification_mode"] + unless ssl_verification_mode.nil? + case ssl_verification_mode + when 'none' + logger.warn "You have enabled encryption but DISABLED certificate verification, " + + "to make sure your data is secure set `ssl_verification_mode => full`" + ssl_options[:verify] = :disable + else + ssl_options[:verify] = :strict + end + end + + ssl_options[:cipher_suites] = params["ssl_cipher_suites"] if params.include?("ssl_cipher_suites") + protocols = params['ssl_supported_protocols'] + ssl_options[:protocols] = protocols if protocols&.any? + ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint + + ssl_options + end + + # @param kind is a string [truststore|keystore] + def setup_client_ssl_store(ssl_options, kind, store_path) + if store_path + ssl_options[kind.to_sym] = store_path + ssl_options["#{kind}_type".to_sym] = params["ssl_#{kind}_type"] if params.include?("ssl_#{kind}_type") + ssl_options["#{kind}_password".to_sym] = params["ssl_#{kind}_password"].value if params.include?("ssl_#{kind}_password") + end + end + def new_client # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ... @@ -290,7 +399,7 @@ def validate_authentication raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key' end - if @api_key && @api_key.value && @ssl != true + if @api_key && @api_key.value && @ssl_enabled != true raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option") end end @@ -353,4 +462,48 @@ def test_connection! raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch" end end + + def setup_ssl_params! + @ssl_enabled = normalize_config(:ssl_enabled) do |normalize| + normalize.with_deprecated_alias(:ssl) + end + + # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set + infer_ssl_enabled_from_hosts + + @ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize| + normalize.with_deprecated_alias(:keystore) + end + + @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize| + normalize.with_deprecated_alias(:keystore_password) + end + + @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize| + normalize.with_deprecated_mapping(:ca_file) do |ca_file| + [ca_file] + end + end + + params['ssl_enabled'] = @ssl_enabled + params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil? + params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil? + params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil? + end + + def infer_ssl_enabled_from_hosts + return if original_params.include?('ssl') || original_params.include?('ssl_enabled') + + @ssl_enabled = params['ssl_enabled'] = effectively_ssl? + end + + def effectively_ssl? + return true if @ssl_enabled + + hosts = Array(@hosts) + return false if hosts.nil? || hosts.empty? + + hosts.all? { |host| host && host.to_s.start_with?("https") } + end + end #class LogStash::Filters::Elasticsearch diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index 4a46ac2..6ea0dbd 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -11,9 +11,6 @@ class ElasticsearchClient attr_reader :client def initialize(logger, hosts, options = {}) - ssl = options.fetch(:ssl, false) - keystore = options.fetch(:keystore, nil) - keystore_password = options.fetch(:keystore_password, nil) user = options.fetch(:user, nil) password = options.fetch(:password, nil) api_key = options.fetch(:api_key, nil) @@ -28,17 +25,10 @@ def initialize(logger, hosts, options = {}) logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('') transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('') - hosts = setup_hosts(hosts, ssl) + ssl_options = options.fetch(:ssl, { :enabled => false }) + ssl_enabled = ssl_options.fetch(:enabled, false) - ssl_options = {} - # set ca_file even if ssl isn't on, since the host can be an https url - ssl_options.update(ssl: true, ca_file: options[:ca_file]) if options[:ca_file] - ssl_options.update(ssl: true, trust_strategy: options[:ssl_trust_strategy]) if options[:ssl_trust_strategy] - if keystore - ssl_options[:keystore] = keystore - logger.debug("Keystore for client certificate", :keystore => keystore) - ssl_options[:keystore_password] = keystore_password.value if keystore_password - end + hosts = setup_hosts(hosts, ssl_enabled) client_options = { hosts: hosts, @@ -59,13 +49,14 @@ def search(params) private - def setup_hosts(hosts, ssl) + def setup_hosts(hosts, ssl_enabled) + hosts = Array(hosts).map { |host| host.to_s } # potential SafeURI#to_s hosts.map do |h| if h.start_with?('http:/', 'https:/') h else host, port = h.split(':') - { host: host, port: port, scheme: (ssl ? 'https' : 'http') } + { host: host, port: port, scheme: (ssl_enabled ? 'https' : 'http') } end end end diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index f5f6f60..a01e41e 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '3.14.0' + s.version = '3.15.0' s.licenses = ['Apache License (2.0)'] s.summary = "Copies fields from previous log events in Elasticsearch to current events " s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -24,8 +24,8 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'elasticsearch', ">= 7.14.0" # LS >= 6.7 and < 7.14 all used version 5.0.5 s.add_runtime_dependency 'manticore', ">= 0.7.1" s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0' + s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0' s.add_development_dependency 'cabin', ['~> 0.6'] s.add_development_dependency 'webrick' - s.add_development_dependency 'logstash-devutils' end diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index 74437e3..904fffc 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -524,7 +524,7 @@ def wait_receive_request end context "with ssl" do - let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) } + let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) } it "should set authorization" do plugin.register @@ -594,7 +594,7 @@ def wait_receive_request describe "ca_trusted_fingerprint" do let(:ca_trusted_fingerprint) { SecureRandom.hex(32) } - let(:config) { {"ca_trusted_fingerprint" => ca_trusted_fingerprint}} + let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint}} subject(:plugin) { described_class.new(config) } @@ -630,8 +630,9 @@ def wait_receive_request let(:config) do { - 'keystore' => keystore_path, - 'keystore_password' => keystore_password, + 'hosts' => 'https://localhost:9200', + 'ssl_keystore_path' => keystore_path, + 'ssl_keystore_password' => keystore_password, } end diff --git a/spec/filters/elasticsearch_ssl_spec.rb b/spec/filters/elasticsearch_ssl_spec.rb new file mode 100644 index 0000000..e1d975d --- /dev/null +++ b/spec/filters/elasticsearch_ssl_spec.rb @@ -0,0 +1,264 @@ +require 'stud/temporary' +require "elasticsearch" +require "logstash/codecs/base" + +describe "SSL options" do + let(:es_client_double) { double("Elasticsearch::Client #{self.inspect}") } + let(:hosts) {["localhost"]} + let(:settings) { { "ssl_enabled" => true, "hosts" => hosts } } + + subject do + require "logstash/filters/elasticsearch" + LogStash::Filters::Elasticsearch.new(settings) + end + + before do + allow(es_client_double).to receive(:close) + allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object) + allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double) + end + + after do + subject.close + end + + context "when ssl_enabled is" do + context "true and there is no https hosts" do + let(:hosts) { %w[http://es01 http://es01] } + + it "should not infer the ssl_enabled value" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true) + expect(subject.params).to match hash_including("ssl_enabled" => true) + end + end + + context "false and cloud_id resolved host is https" do + let(:settings) {{ + "ssl_enabled" => false, + "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==" + }} + + it "should not infer the ssl_enabled value" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false) + expect(subject.params).to match hash_including("ssl_enabled" => false) + end + end + end + + context "when neither ssl nor ssl_enabled is set" do + let(:settings) { super().reject { |k| %w[ssl ssl_enabled].include?(k) } } + + context "and there is no https hosts" do + let(:hosts) { %w[http://es01 http://es01] } + + it "should infer the ssl_enabled value to false" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false) + expect(subject.params).to match hash_including("ssl_enabled" => false) + end + end + + context "and there is https hosts" do + let(:hosts) { %w[https://sec-es01 https://sec-es01] } + + it "should infer the ssl_enabled value to true" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true) + expect(subject.params).to match hash_including("ssl_enabled" => true) + end + end + + context "and hosts have no scheme defined" do + let(:hosts) { %w[es01 es01] } + + it "should infer the ssl_enabled value to false" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false) + expect(subject.params).to match hash_including("ssl_enabled" => false) + end + end + + context "and cloud_id resolved host is https" do + let(:settings) {{ + "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==" + }} + + it "should infer the ssl_enabled value to false" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true) + expect(subject.params).to match hash_including("ssl_enabled" => true) + end + end + end + + context "when ssl_verification_mode" do + context "is set to none" do + let(:settings) { super().merge( + "ssl_verification_mode" => "none", + ) } + + it "should print a warning" do + expect(subject.logger).to receive(:warn).with(/You have enabled encryption but DISABLED certificate verification/).at_least(:once) + allow(subject.logger).to receive(:warn).with(any_args) + + subject.register + end + + it "should pass the flag to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable) + end.and_return(es_client_double) + + subject.register + end + end + + context "is set to full" do + let(:settings) { super().merge( + "ssl_verification_mode" => 'full', + ) } + + it "should pass the flag to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :strict) + end.and_return(es_client_double) + + subject.register + end + end + end + + context "with the conflicting configs" do + context "ssl_certificate_authorities and ssl_truststore_path set" do + let(:ssl_truststore_path) { Stud::Temporary.file.path } + let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path } + let(:settings) { super().merge( + "ssl_truststore_path" => ssl_truststore_path, + "ssl_certificate_authorities" => ssl_certificate_authorities_path + ) } + + after :each do + File.delete(ssl_truststore_path) + File.delete(ssl_certificate_authorities_path) + end + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate_authorities\/ca_file" or "ssl_truststore_path"/) + end + end + + context "ssl_certificate and ssl_keystore_path set" do + let(:ssl_keystore_path) { Stud::Temporary.file.path } + let(:ssl_certificate_path) { Stud::Temporary.file.path } + let(:settings) { super().merge( + "ssl_certificate" => ssl_certificate_path, + "ssl_keystore_path" => ssl_keystore_path + ) } + + after :each do + File.delete(ssl_keystore_path) + File.delete(ssl_certificate_path) + end + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate" or "ssl_keystore_path\/keystore"/) + end + end + end + + context "when configured with Java store files" do + let(:ssl_truststore_path) { Stud::Temporary.file.path } + let(:ssl_keystore_path) { Stud::Temporary.file.path } + + after :each do + File.delete(ssl_truststore_path) + File.delete(ssl_keystore_path) + end + + let(:settings) { super().merge( + "ssl_truststore_path" => ssl_truststore_path, + "ssl_truststore_type" => "jks", + "ssl_truststore_password" => "foo", + "ssl_keystore_path" => ssl_keystore_path, + "ssl_keystore_type" => "jks", + "ssl_keystore_password" => "bar", + "ssl_verification_mode" => "full", + "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + "ssl_supported_protocols" => ["TLSv1.3"] + ) } + + it "should pass the parameters to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including( + :enabled => true, + :keystore => ssl_keystore_path, + :keystore_type => "jks", + :keystore_password => "bar", + :truststore => ssl_truststore_path, + :truststore_type => "jks", + :truststore_password => "foo", + :verify => :strict, + :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + :protocols => ["TLSv1.3"], + ) + end.and_return(es_client_double) + + subject.register + end + end + + context "when configured with certificate files" do + let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path } + let(:ssl_certificate_path) { Stud::Temporary.file.path } + let(:ssl_key_path) { Stud::Temporary.file.path } + let(:settings) { super().merge( + "ssl_certificate_authorities" => [ssl_certificate_authorities_path], + "ssl_certificate" => ssl_certificate_path, + "ssl_key" => ssl_key_path, + "ssl_verification_mode" => "full", + "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + "ssl_supported_protocols" => ["TLSv1.3"] + ) } + + after :each do + File.delete(ssl_certificate_authorities_path) + File.delete(ssl_certificate_path) + File.delete(ssl_key_path) + end + + it "should pass the parameters to the ES client" do + expect(::Elasticsearch::Client).to receive(:new) do |args| + expect(args[:ssl]).to match hash_including( + :enabled => true, + :ca_file => ssl_certificate_authorities_path, + :client_cert => ssl_certificate_path, + :client_key => ssl_key_path, + :verify => :strict, + :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"], + :protocols => ["TLSv1.3"], + ) + end.and_return(es_client_double) + + subject.register + end + + context "and only the ssl_certificate is set" do + let(:settings) { super().reject { |k| "ssl_key".eql?(k) } } + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Using an "ssl_certificate" requires an "ssl_key"/) + end + end + + context "and only the ssl_key is set" do + let(:settings) { super().reject { |k| "ssl_certificate".eql?(k) } } + + it "should raise a configuration error" do + expect { subject.register }.to raise_error(LogStash::ConfigurationError, /An "ssl_certificate" is required when using an "ssl_key"/) + end + end + end +end + diff --git a/spec/filters/integration/elasticsearch_spec.rb b/spec/filters/integration/elasticsearch_spec.rb index bc98190..bc85eec 100644 --- a/spec/filters/integration/elasticsearch_spec.rb +++ b/spec/filters/integration/elasticsearch_spec.rb @@ -29,7 +29,7 @@ let(:config) do config = ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config - config = { 'ca_file' => ca_path }.merge(config) if SECURE_INTEGRATION + config = { 'ssl_certificate_authorities' => ca_path }.merge(config) if SECURE_INTEGRATION config end @@ -92,7 +92,7 @@ context 'setting host:port (and ssl)' do # reproduces GH-155 let(:config) do - super().merge "hosts" => [ESHelper.get_host_port], "ssl" => SECURE_INTEGRATION + super().merge "hosts" => [ESHelper.get_host_port], "ssl_enabled" => SECURE_INTEGRATION end it "works" do @@ -110,9 +110,9 @@ let(:config) do super().merge( "hosts" => [ESHelper.get_host_port], - "keystore" => keystore_path, - "keystore_password" => keystore_password, - "ssl" => true, + "ssl_keystore_path" => keystore_path, + "ssl_keystore_password" => keystore_password, + "ssl_enabled" => true, "fields" => { "this" => "contents", "response" => "four-oh-four" } ) end @@ -132,7 +132,7 @@ let(:config) do bc = super() - bc.delete('ca_file') + bc.delete('ssl_certificate_authorities') bc.merge({ 'ca_trusted_fingerprint' => ca_trusted_fingerprint, 'fields' => { "this" => "contents", "response" => "four-oh-four" }