From 4a8cec5f6188434dd065c36b18492382abf83ae8 Mon Sep 17 00:00:00 2001 From: Ben Harosh Date: Fri, 31 May 2024 16:48:36 -0700 Subject: [PATCH 1/2] PTRENG-6034 - partnership registry deprecation + upgrade sidecar image to 4.3 --- CHANGELOG.md | 11 ++++++++--- README.md | 6 +++++- fluent.conf.xray | 2 +- helm/artifactory-ha-values.yaml | 2 +- helm/artifactory-values.yaml | 4 ++-- helm/xray-values.yaml | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f885450..9d1c465 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All changes to the log analytics integration will be documented in this file. +## [1.0.4] - May 31, 2024 +* [BREAKING] Adding deprecation notice for partnership-pts-observability.jfrog.io docker registry +* FluentD sidecar version bumped to 4.3, to upgrade base image to bitnami/fluentd 1.16.5 +* Minor bug fix to FluentD config - fixing dynamic DataDog host config for logs + ## [1.0.3] - April 23, 2024 * Fix order of request and response content length to match spec @@ -9,7 +14,7 @@ All changes to the log analytics integration will be documented in this file. ## [1.0.2] - April 12, 2024 -* Fluentd version bumped to 4.2, which has latest Fluentd plugins. Resolved PTRENG-5895. +* Fluentd version bumped to 4.2, which has latest Fluentd plugins. Resolved PTRENG-5895 ## [1.0.1] - April 11, 2024 @@ -24,7 +29,7 @@ All changes to the log analytics integration will be documented in this file. ## [0.8.0] - Feb 09, 2022 -* Added call home functionality to artifactory fluent configuration. +* Added call home functionality to artifactory fluent configuration ## [0.7.0] - Oct 20, 2020 @@ -32,7 +37,7 @@ All changes to the log analytics integration will be documented in this file. ## [0.6.0] - Sept 25, 2020 -* [BREAKING] Datadog fluentd configs updated to use JF_PRODUCT_DATA_INTERNAL env. +* [BREAKING] Datadog fluentd configs updated to use JF_PRODUCT_DATA_INTERNAL env ## [0.5.0] - Sept 8, 2020 diff --git a/README.md b/README.md index 1c75048..ec70ad7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The following document describes how to configure Datadog to gather logs, metric ## Versions Supported -This integration is last tested with Artifactory 7.77.8 and Xray 3.92.7 versions. +This integration is last tested with Artifactory 7.84.11 and Xray 3.92.7 versions. ## Table of Contents @@ -142,6 +142,10 @@ The recommended installation method for Kubernetes is to utilize the helm chart | Artifactory HA | helm/artifactory-ha-values.yaml | | Xray | helm/xray-values.yaml | +> [!WARNING] +> +> The old docker registry `partnership-pts-observability.jfrog.io`, which contains older versions of this integration is now deprecated. We'll keep the existing docker images on this old registry until August 1st, 2024. After that date, this registry will no longer be available. Please `helm upgrade` your JFrog kubernetes deployment in order to pull images as specified on the above helm value files, from the new `releases-pts-observability-fluentd.jfrog.io` registry. Please do so in order to avoid `ImagePullBackOff` errors in your deployment once this registry is gone. + Add JFrog Helm repository: ```bash diff --git a/fluent.conf.xray b/fluent.conf.xray index d5b08b2..462c1d3 100644 --- a/fluent.conf.xray +++ b/fluent.conf.xray @@ -136,7 +136,7 @@ include_tag_key true dd_source jfrog_platform service jfrog_xray - host http-intake.logs.datadoghq.com + host "http-intake.logs.#{ENV['DATADOG_API_HOST']}" flush_interval 1s # frequency of the buffer flush diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml index 5e5ff26..7de6f9a 100644 --- a/helm/artifactory-ha-values.yaml +++ b/helm/artifactory-ha-values.yaml @@ -15,7 +15,7 @@ artifactory: name: volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.2" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.3" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml index b1ba450..3e52b5c 100644 --- a/helm/artifactory-values.yaml +++ b/helm/artifactory-values.yaml @@ -15,9 +15,9 @@ artifactory: name: artifactory-volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.2" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.3" imagePullPolicy: "IfNotPresent" - volumeMounts: + volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" name: artifactory-volume env: diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml index f418101..5df376f 100644 --- a/helm/xray-values.yaml +++ b/helm/xray-values.yaml @@ -19,7 +19,7 @@ common: name: data-volume customSidecarContainers: | - name: "xray-platform-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.2" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.3" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.xray.persistence.mountPath }}" From 0d377ce6dd5b5132cd1e9686d7ce985c096ed255 Mon Sep 17 00:00:00 2001 From: Ben Harosh Date: Thu, 6 Jun 2024 17:11:04 -0700 Subject: [PATCH 2/2] PTRENG-6131 - Fix init container helm chart to match official changes --- CHANGELOG.md | 3 +- README.md | 105 ++++++++++++++++++-------------- helm/artifactory-ha-values.yaml | 2 +- helm/artifactory-values.yaml | 2 +- helm/xray-values.yaml | 2 +- 5 files changed, 64 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d1c465..7e1f928 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,11 @@ All changes to the log analytics integration will be documented in this file. -## [1.0.4] - May 31, 2024 +## [1.0.4] - June 6, 2024 * [BREAKING] Adding deprecation notice for partnership-pts-observability.jfrog.io docker registry * FluentD sidecar version bumped to 4.3, to upgrade base image to bitnami/fluentd 1.16.5 * Minor bug fix to FluentD config - fixing dynamic DataDog host config for logs +* Update FluentD sidecar helm charts to match recent changes in JFrog's official charts ## [1.0.3] - April 23, 2024 diff --git a/README.md b/README.md index ec70ad7..4be1b85 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This integration is last tested with Artifactory 7.84.11 and Xray 3.92.7 version `Note! You must follow the order of the steps throughout Datadog Configuration` -1. [Datadog Setup](#datadog-setup) +1. [DataDog Setup](#datadog-setup) 2. [JFrog Metrics Setup](#jfrog-metrics-setup) 3. [Fluentd Installation](#fluentd-installation) * [OS / Virtual Machine](#os--virtual-machine) @@ -19,18 +19,15 @@ This integration is last tested with Artifactory 7.84.11 and Xray 3.92.7 version 4. [Dashboards](#dashboards) 5. [References](#references) -## Datadog Setup +## DataDog Setup -Datadog setup can be done by going through the below onboarding steps or by using apiKey directly if one exists. If an apiKey exists, skip the steps below and move on to [Fluentd Installation](#fluentd-installation) to forward logs directly to your datadog account. +DataDog setup for this integration can be done by going through the step below to add a new DataDog apiKey or by using an existing DataDog apiKey.
+If a DataDog apiKey already exists and can be used for this integration, skip this part and move on to [Fluentd Installation](#fluentd-installation) to forward logs and metrics to your DataDog account. -* Create an account in Datadog -* Run the datadog agent in your kubernetes cluster by deploying it with a helm chart -* To enable log collection, update datadog-values.yaml file given in the onboarding steps of datadog -* Once the agent starts reporting, you'll get an apiKey which we'll be using to send formatted logs through fluentd +If you don't have a DataDog apiKey: -Once datadog is setup, we can access logs via Logs > Search. We can also select the specific source that we want to get logs from. Adding proper metadata is the key to unlocking the full potential of your logs in datadog. By default, the hostname and timestamp fields should be remapped. - -* Add all attributes as facets from Facets > Add on the left side of the screen in Logs > search +* Create an account in DataDog, if one doesn't exist +* Follow the official DataDog instructions [here](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token) how to generate an apiKey that will be used in the following sections ## JFrog Metrics Setup @@ -143,7 +140,7 @@ The recommended installation method for Kubernetes is to utilize the helm chart | Xray | helm/xray-values.yaml | > [!WARNING] -> +> > The old docker registry `partnership-pts-observability.jfrog.io`, which contains older versions of this integration is now deprecated. We'll keep the existing docker images on this old registry until August 1st, 2024. After that date, this registry will no longer be available. Please `helm upgrade` your JFrog kubernetes deployment in order to pull images as specified on the above helm value files, from the new `releases-pts-observability-fluentd.jfrog.io` registry. Please do so in order to avoid `ImagePullBackOff` errors in your deployment once this registry is gone. Add JFrog Helm repository: @@ -185,56 +182,65 @@ export MASTER_KEY=$(openssl rand -hex 32) --set artifactory.license.dataKey=artifactory.cluster.license \ --set artifactory.metrics.enabled=true \ --set artifactory.openMetrics.enabled=true \ - -n $INST_NAMESPACE + -n $INST_NAMESPACE --create-namespace ``` :bulb: Metrics collection is disabled by default in Artifactory. Please make sure that you are following the above `helm upgrade` command to enable them in Artifactory by setting to true both `artifactory.metrics.enabled` and `artifactory.openMetrics.enabled` - 2. Create a secret for JFrog's admin token - [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) using any of the following methods + Get the ip address of the newly deployed Artifactory: ```bash - kubectl create secret generic jfrog-admin-token --from-file=token= + export SERVICE_IP=$(kubectl get svc -n $INST_NAMESPACE artifactory-artifactory-nginx -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP + ``` - OR +2. Create a secret for JFrog's admin token - [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) using any of the following methods - kubectl create secret generic jfrog-admin-token --from-literal=token= - ``` - 3. For Artifactory installation, download the .env file from [here](https://github.com/jfrog/log-analytics-datadog/raw/master/helm/jfrog_helm.env). Fill in the jfrog_helm.env file with correct values. + ```bash + kubectl create secret generic jfrog-admin-token --from-file=token= - * **JF_PRODUCT_DATA_INTERNAL**: Helm based installs will already have this defined based upon the underlying Docker images. Not a required field for k8s installation - * **DATADOG_API_KEY**: API Key from [Datadog](https://app.datadoghq.com/organization-settings/api-keys) - * **DATADOG_API_HOST**: Your DataDog host based on your [DataDog Site Parameter from this list](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site) - * **JPD_URL**: Artifactory JPD URL of the format `http://` - * **JPD_ADMIN_USERNAME**: Artifactory username for authentication - * **COMMON_JPD**: This flag should be set as true only for non-Kubernetes installations or installations where the JPD base URL is the same to access both Artifactory and Xray (for example, `https://sample_base_url/artifactory` or `https://sample_base_url/xray`) + OR - Apply the .env files using the helm command below + kubectl create secret generic jfrog-admin-token --from-literal=token= + ``` - ```shell - source jfrog_helm.env - ``` - 4. Postgres password is required to upgrade Artifactory. Run the following command to get the current password +3. For Artifactory installation, download the .env file from [here](https://github.com/jfrog/log-analytics-datadog/raw/master/helm/jfrog_helm.env). Fill in the jfrog_helm.env file with correct values. - ```bash - POSTGRES_PASSWORD=$(kubectl get secret artifactory-postgresql -n $INST_NAMESPACE -o jsonpath="{.data.postgresql-password}" | base64 --decode) - ``` - 5. Upgrade Artifactory installation using the command below + * **JF_PRODUCT_DATA_INTERNAL**: Helm based installs will already have this defined based upon the underlying Docker images. Not a required field for k8s installation + * **DATADOG_API_KEY**: API Key from [Datadog](https://app.datadoghq.com/organization-settings/api-keys) + * **DATADOG_API_HOST**: Your DataDog host based on your [DataDog Site Parameter from this list](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site) + * **JPD_URL**: Artifactory JPD URL of the format `http://` + * **JPD_ADMIN_USERNAME**: Artifactory username for authentication + * **COMMON_JPD**: This flag should be set as true only for non-Kubernetes installations or installations where the JPD base URL is the same to access both Artifactory and Xray (for example, `https://sample_base_url/artifactory` or `https://sample_base_url/xray`) - ```bash - helm upgrade --install artifactory jfrog/artifactory \ - --set artifactory.masterKey=$MASTER_KEY \ - --set artifactory.joinKey=$JOIN_KEY \ - --set artifactory.metrics.enabled=true --set artifactory.openMetrics.enabled=true \ - --set databaseUpgradeReady=true --set postgresql.postgresqlPassword=$POSTGRES_PASSWORD --set nginx.service.ssloffload=true \ - --set datadog.api_key=$DATADOG_API_KEY \ - --set datadog.api_host=$DATADOG_API_HOST \ - --set jfrog.observability.jpd_url=$JPD_URL \ - --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ - --set jfrog.observability.common_jpd=$COMMON_JPD \ - -f helm/artifactory-values.yaml \ - -n $INST_NAMESPACE + Apply the .env files using the helm command below + + ```bash + source jfrog_helm.env ``` +4. Postgres password is required to upgrade Artifactory. Run the following command to get the current password + + ```bash + POSTGRES_PASSWORD=$(kubectl get secret artifactory-postgresql -n $INST_NAMESPACE -o jsonpath="{.data.postgresql-password}" | base64 --decode) + ``` +5. Upgrade Artifactory installation using the command below + + ```bash + helm upgrade --install artifactory jfrog/artifactory \ + --set artifactory.masterKey=$MASTER_KEY \ + --set artifactory.joinKey=$JOIN_KEY \ + --set artifactory.metrics.enabled=true --set artifactory.openMetrics.enabled=true \ + --set databaseUpgradeReady=true --set postgresql.postgresqlPassword=$POSTGRES_PASSWORD --set nginx.service.ssloffload=true \ + --set datadog.api_key=$DATADOG_API_KEY \ + --set datadog.api_host=$DATADOG_API_HOST \ + --set jfrog.observability.jpd_url=$JPD_URL \ + --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ + --set jfrog.observability.common_jpd=$COMMON_JPD \ + -f helm/artifactory-values.yaml \ + -n $INST_NAMESPACE + ``` + #### Artifactory-HA ⎈: 1. For HA installation, please create a license secret on your cluster prior to installation. @@ -256,6 +262,13 @@ export MASTER_KEY=$(openssl rand -hex 32) ``` :bulb: Metrics collection is disabled by default in Artifactory-HA. Please make sure that you are following the above `helm upgrade` command to enable them in Artifactory-HA by setting to true both `artifactory.metrics.enabled` and `artifactory.openMetrics.enabled` + + Get the ip address of the newly deployed Artifactory: + + ```bash + export SERVICE_IP=$(kubectl get svc -n $INST_NAMESPACE artifactory-artifactory-nginx -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP + ``` 3. Create a secret for JFrog's admin token - [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) using any of the following methods ```bash diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml index 7de6f9a..4c5b8f4 100644 --- a/helm/artifactory-ha-values.yaml +++ b/helm/artifactory-ha-values.yaml @@ -2,7 +2,7 @@ installerInfo: '{ "productId": "OnPremObservability-Datadog/1.0.0", "features": artifactory: customInitContainersBegin: | - name: "prepare-fluentd-conf-on-persistent-volume" - image: "{{ .Values.initContainerImage }}" + image: {{ include "artifactory.getImageInfoByValue" (list . "initContainers") }} imagePullPolicy: "{{ .Values.artifactory.image.pullPolicy }}" command: - 'sh' diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml index 3e52b5c..17bf9ea 100644 --- a/helm/artifactory-values.yaml +++ b/helm/artifactory-values.yaml @@ -2,7 +2,7 @@ installerInfo: '{ "productId": "OnPremObservability-Datadog/1.0.0", "features": artifactory: customInitContainersBegin: | - name: "download-fluentd-conf-on-persistent-volume" - image: "{{ .Values.initContainerImage }}" + image: {{ include "artifactory.getImageInfoByValue" (list . "initContainers") }} imagePullPolicy: "{{ .Values.artifactory.image.pullPolicy }}" command: - 'sh' diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml index 5df376f..efeca5f 100644 --- a/helm/xray-values.yaml +++ b/helm/xray-values.yaml @@ -6,7 +6,7 @@ replicaCount: 1 common: customInitContainersBegin: | - name: "download-fluentd-conf-on-persistent-volume" - image: "{{ .Values.initContainerImage }}" + image: {{ include "xray.getImageInfoByValue" (list . "initContainers") }} imagePullPolicy: "{{ .Values.imagePullPolicy }}" command: - 'sh'