Merge pull request #37 from jfrog/Pipeline_changes

Datadog Pipeline changes
jfrog · Sep 11, 2023 · 492f8a0 · 492f8a0
2 parents 8b9baa4 + 22f879f
commit 492f8a0
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 62 deletions.
diff --git a/fluent.conf.rt b/fluent.conf.rt
@@ -116,16 +116,6 @@
     @type none
   </parse>
 </source>
-<source>
-  @type tail
-  @id frontend_service_tail
-  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log"
-  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos"
-  tag jfrog.rt.frontend.service
-  <parse>
-    @type none
-  </parse>
-</source>
 <source>
   @type tail
   @id metadata_service_tail
@@ -157,8 +147,6 @@
       @type multiline
       format_firstline /\d{4}-\d{1,2}-\d{1,2}/
       format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
-      time_key log_timestamp
-      time_format %Y-%m-%dT%H:%M:%S.%LZ
   </parse>
 </source>
 ## REQUEST LOGS
@@ -171,8 +159,6 @@
   <parse>
     @type regexp
     expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
-    time_key log_timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%LZ
     types response_content_length:integer, request_content_length:integer, return_status:integer
   </parse>
 </source>
@@ -213,9 +199,7 @@
   pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
   tag jfrog.rt.router.request
   <parse>
-    @type json
-    time_key time
-    time_format %Y-%m-%dT%H:%M:%S%:z
+    @type none
   </parse>
 </source>
 <source>
@@ -224,11 +208,10 @@
   path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log"
   pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log.pos"
   tag jfrog.rt.request.out
+  read_from_head true
   <parse>
     @type regexp
     expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_repo_name>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<remote_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)$
-    time_key log_timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%LZ
   </parse>
 </source>
 ## ACCESS LOG
@@ -241,8 +224,6 @@
   <parse>
     @type regexp
     expression /^(?<log_timestamp>[^ ]*) \[(?<trace_id>[^\]]*)\] \[(?<action_response>[^\]]*)\] (?<repo_path>.*) for client : (?<username>.+)\/(?<ip>\s*\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\.(?<message>.+)?$/
-    time_key log_timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%LZ
   </parse>
 </source>
 ## AUDIT LOG
@@ -255,8 +236,6 @@
   <parse>
     @type regexp
     expression /^(?<log_timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
-    time_key log_timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%LZ
   </parse>
 </source>
 
@@ -290,8 +269,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.rt.artifactory.service>
@@ -301,8 +278,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.rt.metadata.service>
@@ -312,8 +287,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.rt.router.service>
@@ -323,8 +296,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   ## REQUEST LOGS
@@ -334,8 +305,6 @@
     <parse>
       @type regexp
       expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
-      time_key log_timestamp
-      time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.rt.artifactory.request>
@@ -360,8 +329,6 @@
     <parse>
       @type regexp
       expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
-      time_key log_timestamp
-      time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.rt.metadata.request>
@@ -370,8 +337,6 @@
     <parse>
       @type regexp
       expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
-      time_key log_timestamp
-      time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   ## ACCESS LOGS
@@ -403,7 +368,8 @@
     api_key "#{ENV['DATADOG_API_KEY']}"
     #optional
     include_tag_key true
-    dd_source jfrog_artifactory
+    dd_source jfrog_platform
+    service jfrog_artifactory
     host http-intake.logs.datadoghq.com
     <buffer>
       flush_interval 1s

diff --git a/fluent.conf.xray b/fluent.conf.xray
@@ -134,7 +134,8 @@
   api_key "#{ENV['DATADOG_API_KEY']}"
   #optional
   include_tag_key true
-  dd_source jfrog_xray
+  dd_source jfrog_platform
+  service jfrog_xray
   host http-intake.logs.datadoghq.com
   <buffer>
     flush_interval 1s
@@ -208,8 +209,6 @@
   <parse>
       @type regexp
       expression ^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.+)$
-      time_key log_timestamp
-      time_format %Y-%m-%dT%H:%M:%S.%LZ
   </parse>
 </source>
 # REQUEST LOGS
@@ -220,9 +219,7 @@
   pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
   tag jfrog.xray.router.request
   <parse>
-    @type json
-    time_key time
-    time_format %Y-%m-%dT%H:%M:%S%:z
+    @type none
   </parse>
 </source>
 <source>
@@ -234,8 +231,6 @@
   <parse>
     @type regexp
     expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^ ]*)\|(?<remote_address>[^|]++)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>.*)$
-    time_key log_timestamp
-    time_format %Y-%m-%dT%H:%M:%S.%LZ
   </parse>
 </source>
 
@@ -269,8 +264,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
     emit_invalid_record_to_error false
   </filter>
@@ -281,8 +274,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
     emit_invalid_record_to_error false
   </filter>
@@ -293,8 +284,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.xray.router.service>
@@ -304,8 +293,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   <filter jfrog.xray.server.service>
@@ -315,8 +302,6 @@
         @type multiline
         format_firstline /\d{4}-\d{1,2}-\d{1,2}/
         format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
-        time_key log_timestamp
-        time_format %Y-%m-%dT%H:%M:%S.%LZ
     </parse>
   </filter>
   ## ALL LOGS
@@ -335,7 +320,8 @@
     api_key "#{ENV['DATADOG_API_KEY']}"
     #optional
     include_tag_key true
-    dd_source jfrog_xray
+    dd_source jfrog_platform
+    service jfrog_xray
     host http-intake.logs.datadoghq.com
     <buffer>
       flush_interval 1s

diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml
@@ -9,13 +9,13 @@ artifactory:
         - '-c'
         - >
           mkdir -p {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/;
-          curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/fluent.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
+          curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/Pipeline_changes/fluent.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
       volumeMounts:
         - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
           name: volume
   customSidecarContainers: |
     - name: "artifactory-fluentd-sidecar"
-      image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0"
+      image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0"
       imagePullPolicy: "IfNotPresent"
       volumeMounts:
         - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"

diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml
@@ -15,7 +15,7 @@ artifactory:
           name: artifactory-volume
   customSidecarContainers: |
     - name: "artifactory-fluentd-sidecar"
-      image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0"
+      image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0"
       imagePullPolicy: "IfNotPresent"
       volumeMounts:
         - mountPath: "{{ .Values.artifactory.persistence.mountPath }}"

diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml
@@ -13,13 +13,13 @@ common:
         - '-c'
         - >
           mkdir -p {{ .Values.xray.persistence.mountPath }}/etc/fluentd/;
-          curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/fluent.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf;
+          curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/Pipeline_changes/fluent.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf;
       volumeMounts:
         - mountPath: "{{ .Values.xray.persistence.mountPath }}"
           name: data-volume
   customSidecarContainers: |
     - name: "xray-platform-fluentd-sidecar"
-      image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0"
+      image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0"
       imagePullPolicy: "IfNotPresent"
       volumeMounts:
         - mountPath: "{{ .Values.xray.persistence.mountPath }}"