diff --git a/CHANGELOG.md b/CHANGELOG.md index 1279fd9..08481b3 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All changes to the log analytics integration will be documented in this file. +## [1.0.9] - October 25, 2024 + +* Add support for metrics outbound payload compression, with `gzip_compression` FluentD param in `jfrog_send_metrics` plugin +* Add support for a configurable http request timeout, with `request_timeout` FluentD param in `jfrog_metrics` and `jfrog_send_metrics` plugins +* FluentD sidecar version bumped to 4.9, to incorporate the above changes +* Add configuration support via environment variable for `verify_ssl` FluentD flag + ## [1.0.8] - September 12, 2024 * FluentD sidecar image version bumped to 4.8, to add verify_ssl flag support for JFrog's FluentD metrics plugins diff --git a/README.md b/README.md index 1ab187d..61cb9e2 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ We rely on environment variables to stream log files to your observability dashb * **JF_PRODUCT_DATA_INTERNAL**: The environment variable JF_PRODUCT_DATA_INTERNAL must be defined to the correct location. For each JFrog service, you can find its active log files in the `$JFROG_HOME//var/log` directory * **DATADOG_API_KEY**: API Key from [Datadog](https://app.datadoghq.com/organization-settings/api-keys) * **DATADOG_API_HOST**: Your DataDog host based on your [DataDog Site Parameter from this list](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site) +* **DATADOG_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) +* **DATADOG_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to DataDog * **JPD_URL**: Artifactory JPD URL with the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **JFROG_ADMIN_TOKEN**: Artifactory [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) for authentication @@ -121,6 +123,8 @@ In order to run FluentD as a docker image to send the logs, violations, and metr * **JF_PRODUCT_DATA_INTERNAL**: The environment variable JF_PRODUCT_DATA_INTERNAL must be defined to the correct location. For each JFrog service you will find its active log files in the `$JFROG_HOME//var/log` directory * **DATADOG_API_KEY**: API Key from [Datadog](https://docs.datadoghq.com/account_management/api-app-keys/) * **DATADOG_API_HOST**: Your DataDog host based on your [DataDog Site Parameter from this list](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site) + * **DATADOG_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) + * **DATADOG_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to DataDog * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **JFROG_ADMIN_TOKEN**: Artifactory [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) for authentication @@ -211,6 +215,8 @@ export MASTER_KEY=$(openssl rand -hex 32) * **JF_PRODUCT_DATA_INTERNAL**: Helm based installs will already have this defined based upon the underlying Docker images. Not a required field for k8s installation * **DATADOG_API_KEY**: API Key from [Datadog](https://app.datadoghq.com/organization-settings/api-keys) * **DATADOG_API_HOST**: Your DataDog host based on your [DataDog Site Parameter from this list](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site) + * **DATADOG_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) + * **DATADOG_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to DataDog * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **COMMON_JPD**: This flag should be set as true only for non-Kubernetes installations or installations where the JPD base URL is the same to access both Artifactory and Xray (for example, `https://sample_base_url/artifactory` or `https://sample_base_url/xray`) @@ -233,6 +239,8 @@ export MASTER_KEY=$(openssl rand -hex 32) --set databaseUpgradeReady=true --set postgresql.postgresqlPassword=$POSTGRES_PASSWORD --set nginx.service.ssloffload=true \ --set datadog.api_key=$DATADOG_API_KEY \ --set datadog.api_host=$DATADOG_API_HOST \ + --set datadog.compress_data=$DATADOG_COMPRESS_DATA \ + --set datadog.verify_ssl=$DATADOG_VERIFY_SSL \ --set jfrog.observability.jpd_url=$JPD_URL \ --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ --set jfrog.observability.common_jpd=$COMMON_JPD \ @@ -281,6 +289,8 @@ export MASTER_KEY=$(openssl rand -hex 32) * **JF_PRODUCT_DATA_INTERNAL**: Helm based installs will already have this defined based upon the underlying Docker images. Not a required field for k8s installation * **DATADOG_API_KEY**: API Key from [Datadog](https://app.datadoghq.com/organization-settings/api-keys) * **DATADOG_API_HOST**: Your DataDog host based on your [DataDog Site Parameter from this list](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site) + * **DATADOG_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) + * **DATADOG_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to DataDog * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **COMMON_JPD**: This flag should be set as true only for non-Kubernetes installations or installations where the JPD base URL is the same to access both Artifactory and Xray (for example, `https://sample_base_url/artifactory` or `https://sample_base_url/xray`) @@ -304,6 +314,8 @@ export MASTER_KEY=$(openssl rand -hex 32) --set databaseUpgradeReady=true --set postgresql.postgresqlPassword=$POSTGRES_PASSWORD --set nginx.service.ssloffload=true \ --set datadog.api_key=$DATADOG_API_KEY \ --set datadog.api_host=$DATADOG_API_HOST \ + --set datadog.compress_data=$DATADOG_COMPRESS_DATA \ + --set datadog.verify_ssl=$DATADOG_VERIFY_SSL \ --set jfrog.observability.jpd_url=$JPD_URL \ --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ --set jfrog.observability.common_jpd=$COMMON_JPD \ @@ -327,6 +339,8 @@ For Xray installation, download the .env file from [here](https://raw.githubuser * **JF_PRODUCT_DATA_INTERNAL**: Helm based installs will already have this defined based upon the underlying Docker images. Not a required field for k8s installation * **DATADOG_API_KEY**: API Key from [Datadog](https://app.datadoghq.com/organization-settings/api-keys) +* **DATADOG_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) +* **DATADOG_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to DataDog * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **COMMON_JPD**: This flag should be set as true only for non-Kubernetes installations or installations where the JPD base URL is the same to access both Artifactory and Xray (for example, `https://sample_base_url/artifactory` or `https://sample_base_url/xray`) @@ -351,6 +365,8 @@ helm upgrade --install xray jfrog/xray --set xray.jfrogUrl=$JPD_URL \ --set xray.joinKey=$JOIN_KEY \ --set datadog.api_key=$DATADOG_API_KEY \ --set datadog.api_host=$DATADOG_API_HOST \ + --set datadog.compress_data=$DATADOG_COMPRESS_DATA \ + --set datadog.verify_ssl=$DATADOG_VERIFY_SSL \ --set jfrog.observability.jpd_url=$JPD_URL \ --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ --set jfrog.observability.common_jpd=$COMMON_JPD \ diff --git a/docker-build/docker.env b/docker-build/docker.env index c15f636..4273dcc 100644 --- a/docker-build/docker.env +++ b/docker-build/docker.env @@ -1,6 +1,8 @@ JF_PRODUCT_DATA_INTERNAL=path_to_jfrog_logs DATADOG_API_KEY=change_me DATADOG_API_HOST=datadoghq.com +DATADOG_VERIFY_SSL=change_me +DATADOG_COMPRESS_DATA=change_me JPD_URL=https://change_me.jfrog.io JPD_ADMIN_USERNAME=admin JFROG_ADMIN_TOKEN=change_me diff --git a/fluent.conf.rt b/fluent.conf.rt index a82570e..f58c2c1 100644 --- a/fluent.conf.rt +++ b/fluent.conf.rt @@ -9,15 +9,21 @@ token "#{ENV['JFROG_ADMIN_TOKEN']}" common_jpd "#{ENV['COMMON_JPD']}" target_platform "DATADOG" + execution_interval 60s + request_timeout 30s + # verify_ssl "#{ENV['DATADOG_VERIFY_SSL']}" @type jfrog_send_metrics target_platform "DATADOG" apikey "#{ENV['DATADOG_API_KEY']}" url "https://api.#{ENV['DATADOG_API_HOST']}/api/v2/series" + gzip_compression "#{ENV['DATADOG_COMPRESS_DATA']}" + request_timeout 30s + # verify_ssl "#{ENV['DATADOG_VERIFY_SSL']}" # ddtags ["instance:test-artifactory", "cluster:GKE"] -# ALL CALLHOME +# CALLHOME @type exec tag callhome diff --git a/fluent.conf.xray b/fluent.conf.xray index 462c1d3..e784392 100644 --- a/fluent.conf.xray +++ b/fluent.conf.xray @@ -9,12 +9,17 @@ token "#{ENV['JFROG_ADMIN_TOKEN']}" common_jpd "#{ENV['COMMON_JPD']}" target_platform "DATADOG" + execution_interval 60s + request_timeout 30s + # verify_ssl "#{ENV['DATADOG_VERIFY_SSL']}" @type jfrog_send_metrics target_platform "DATADOG" apikey "#{ENV['DATADOG_API_KEY']}" url "https://api.#{ENV['DATADOG_API_HOST']}/api/v2/series" + gzip_compression "#{ENV['DATADOG_COMPRESS_DATA']}" + # verify_ssl "#{ENV['DATADOG_VERIFY_SSL']}" # ddtags ["instance:test-artifactory", "cluster:GKE"] # ALL CALLHOME diff --git a/fluentd-installer/Dockerfile.fluentd.sidecar b/fluentd-installer/Dockerfile.fluentd.sidecar index f2b9bc8..5cd489b 100644 --- a/fluentd-installer/Dockerfile.fluentd.sidecar +++ b/fluentd-installer/Dockerfile.fluentd.sidecar @@ -4,15 +4,10 @@ LABEL maintainer "Partner Engineering " USER root -##Uninstall elastic plugin which is preinstalled in bitnami fluentd -##Pin elastic gem version to 7.14 -RUN fluent-gem uninstall elasticsearch -a --ignore-dependencies -RUN fluent-gem install elasticsearch -v 7.14 --no-document ## Install custom Fluentd plugins RUN fluent-gem install fluent-plugin-jfrog-siem --no-document RUN fluent-gem install fluent-plugin-splunk-hec --no-document RUN fluent-gem install fluent-plugin-datadog --no-document -RUN fluent-gem install fluent-plugin-elasticsearch --no-document RUN fluent-gem install fluent-plugin-record-modifier --no-document RUN fluent-gem install fluent-plugin-jfrog-metrics --no-document RUN fluent-gem install fluent-plugin-jfrog-send-metrics --no-document @@ -20,6 +15,4 @@ RUN fluent-gem install fluent-plugin-newrelic --no-document RUN fluent-gem install fluent-plugin-concat --no-document RUN fluent-gem uninstall concurrent-ruby -v '>1.1.9' - - USER 1001 diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml index f83ac45..5aed26b 100644 --- a/helm/artifactory-ha-values.yaml +++ b/helm/artifactory-ha-values.yaml @@ -18,7 +18,7 @@ artifactory: name: volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.8" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.9" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" @@ -41,11 +41,17 @@ artifactory: value: {{ .Values.datadog.api_key }} - name: DATADOG_API_HOST value: {{ .Values.datadog.api_host }} + - name: DATADOG_VERIFY_SSL + value: {{ .Values.datadog.verify_ssl | quote}} + - name: DATADOG_COMPRESS_DATA + value: {{ .Values.datadog.compress_data | quote }} - name: FLUENTD_CONF value: ../../../..{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf datadog: api_key: DATADOG_API_KEY api_host: DATADOG_API_HOST + compress_data: DATADOG_COMPRESS_DATA + verify_ssl: DATADOG_VERIFY_SSL jfrog: observability: jpd_url: JPD_URL diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml index a351884..994e6fa 100644 --- a/helm/artifactory-values.yaml +++ b/helm/artifactory-values.yaml @@ -18,7 +18,7 @@ artifactory: name: artifactory-volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.8" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.9" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" @@ -41,11 +41,17 @@ artifactory: value: {{ .Values.datadog.api_key }} - name: DATADOG_API_HOST value: {{ .Values.datadog.api_host }} + - name: DATADOG_VERIFY_SSL + value: {{ .Values.datadog.verify_ssl | quote}} + - name: DATADOG_COMPRESS_DATA + value: {{ .Values.datadog.compress_data | quote }} - name: FLUENTD_CONF value: ../../../..{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf datadog: api_key: DATADOG_API_KEY api_host: DATADOG_API_HOST + compress_data: DATADOG_COMPRESS_DATA + verify_ssl: DATADOG_VERIFY_SSL jfrog: observability: jpd_url: JPD_URL diff --git a/helm/jfrog_helm.env b/helm/jfrog_helm.env index f8aed1d..5481033 100644 --- a/helm/jfrog_helm.env +++ b/helm/jfrog_helm.env @@ -1,5 +1,7 @@ export DATADOG_API_KEY=DATADOG_API_KEY export DATADOG_API_HOST=datadoghq.com +export DATADOG_COMPRESS_DATA=true +export DATADOG_VERIFY_SSL=true export JPD_URL=http://abc.jfrog.io export JPD_ADMIN_USERNAME=admin export COMMON_JPD=false diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml index 7e8e5aa..7f35126 100644 --- a/helm/xray-values.yaml +++ b/helm/xray-values.yaml @@ -19,7 +19,7 @@ common: name: data-volume customSidecarContainers: | - name: "xray-platform-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.8" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.9" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.xray.persistence.mountPath }}" @@ -44,9 +44,15 @@ common: value: {{ .Values.datadog.api_key }} - name: DATADOG_API_HOST value: {{ .Values.datadog.api_host }} + - name: DATADOG_VERIFY_SSL + value: {{ .Values.datadog.verify_ssl | quote}} + - name: DATADOG_COMPRESS_DATA + value: {{ .Values.datadog.compress_data | quote }} datadog: api_key: DATADOG_API_KEY api_host: DATADOG_API_HOST + compress_data: DATADOG_COMPRESS_DATA + verify_ssl: DATADOG_VERIFY_SSL jfrog: observability: jpd_url: JPD_URL diff --git a/jfrog.env b/jfrog.env index 0a5238b..c73983a 100644 --- a/jfrog.env +++ b/jfrog.env @@ -1,7 +1,10 @@ export JF_PRODUCT_DATA_INTERNAL=JF_PRODUCT_DATA_INTERNAL export DATADOG_API_KEY=DATADOG_API_KEY export DATADOG_API_HOST=datadoghq.com +export DATADOG_COMPRESS_DATA=true +export DATADOG_VERIFY_SSL=true export JPD_URL=http://abc.jfrog.io export JPD_ADMIN_USERNAME=admin export JFROG_ADMIN_TOKEN=JFROG_ADMIN_TOKEN export COMMON_JPD=false +