diff --git a/fluent.conf.rt b/fluent.conf.rt index cba5d1b..db857e1 100644 --- a/fluent.conf.rt +++ b/fluent.conf.rt @@ -116,16 +116,6 @@ @type none - - @type tail - @id frontend_service_tail - path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log" - pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos" - tag jfrog.rt.frontend.service - - @type none - - @type tail @id metadata_service_tail @@ -157,8 +147,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] \[\] -(?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ ## REQUEST LOGS @@ -171,8 +159,6 @@ @type regexp expression ^(?[^ ]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ types response_content_length:integer, request_content_length:integer, return_status:integer @@ -213,9 +199,7 @@ pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos" tag jfrog.rt.router.request - @type json - time_key time - time_format %Y-%m-%dT%H:%M:%S%:z + @type none @@ -224,11 +208,10 @@ path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log" pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log.pos" tag jfrog.rt.request.out + read_from_head true @type regexp expression ^(?[^ ]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ ## ACCESS LOG @@ -241,8 +224,6 @@ @type regexp expression /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] (?.*) for client : (?.+)\/(?\s*\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\.(?.+)?$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ ## AUDIT LOG @@ -255,8 +236,6 @@ @type regexp expression /^(?[^ ]*)\|(?[^ ]*)\|(?[^ ]*)\|(?[^ ]*)\|(?[^ ]*)\|(?[^ ]*)\|(?[^ ]*)\|(?[^ ]*)\|(?.*)/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -290,8 +269,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] -(?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -301,8 +278,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] -(?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -312,8 +287,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] \[\] -(?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -323,8 +296,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] \[\] -(?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ ## REQUEST LOGS @@ -334,8 +305,6 @@ @type regexp expression ^(?[^ ]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -360,8 +329,6 @@ @type regexp expression ^(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -370,8 +337,6 @@ @type regexp expression ^(?[^ ]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.+)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ ## ACCESS LOGS @@ -403,7 +368,8 @@ api_key "#{ENV['DATADOG_API_KEY']}" #optional include_tag_key true - dd_source jfrog_artifactory + dd_source jfrog_platform + service jfrog_artifactory host http-intake.logs.datadoghq.com flush_interval 1s diff --git a/fluent.conf.xray b/fluent.conf.xray index 147d8c0..70bfe30 100644 --- a/fluent.conf.xray +++ b/fluent.conf.xray @@ -134,7 +134,8 @@ api_key "#{ENV['DATADOG_API_KEY']}" #optional include_tag_key true - dd_source jfrog_xray + dd_source jfrog_platform + service jfrog_xray host http-intake.logs.datadoghq.com flush_interval 1s @@ -208,8 +209,6 @@ @type regexp expression ^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] -(?.+)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ # REQUEST LOGS @@ -220,9 +219,7 @@ pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos" tag jfrog.xray.router.request - @type json - time_key time - time_format %Y-%m-%dT%H:%M:%S%:z + @type none @@ -234,8 +231,6 @@ @type regexp expression ^(?[^ ]*)\|(?[^ ]*)\|(?[^|]++)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?[^\|]*)\|(?.*)$ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -269,8 +264,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] (?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ emit_invalid_record_to_error false @@ -281,8 +274,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] (?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ emit_invalid_record_to_error false @@ -293,8 +284,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] (?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -304,8 +293,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] \[\] -(?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ @@ -315,8 +302,6 @@ @type multiline format_firstline /\d{4}-\d{1,2}-\d{1,2}/ format1 /^(?[^ ]*) \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?[^\]]*)\] \[(?.*)\] \[(?.*)\] (?.*)$/ - time_key log_timestamp - time_format %Y-%m-%dT%H:%M:%S.%LZ ## ALL LOGS @@ -335,7 +320,8 @@ api_key "#{ENV['DATADOG_API_KEY']}" #optional include_tag_key true - dd_source jfrog_xray + dd_source jfrog_platform + service jfrog_xray host http-intake.logs.datadoghq.com flush_interval 1s diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml index a0364af..5907710 100644 --- a/helm/artifactory-ha-values.yaml +++ b/helm/artifactory-ha-values.yaml @@ -9,13 +9,13 @@ artifactory: - '-c' - > mkdir -p {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/; - curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/fluent.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf + curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/Pipeline_changes/fluent.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" name: volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml index 86fcd57..9fe6bb8 100644 --- a/helm/artifactory-values.yaml +++ b/helm/artifactory-values.yaml @@ -15,7 +15,7 @@ artifactory: name: artifactory-volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml index 6245f83..f1d2310 100644 --- a/helm/xray-values.yaml +++ b/helm/xray-values.yaml @@ -13,13 +13,13 @@ common: - '-c' - > mkdir -p {{ .Values.xray.persistence.mountPath }}/etc/fluentd/; - curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/fluent.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf; + curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/Pipeline_changes/fluent.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf; volumeMounts: - mountPath: "{{ .Values.xray.persistence.mountPath }}" name: data-volume customSidecarContainers: | - name: "xray-platform-fluentd-sidecar" - image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.xray.persistence.mountPath }}"