Merge pull request #36 from naver/add-customize_log_export_dir

Add PathTemplate to customize the directory for exporting logs
naver · Jan 10, 2025 · 0d87951 · 0d87951
2 parents c59fbe9 + 40d0439
commit 0d87951
Show file tree

Hide file tree

Showing 15 changed files with 426 additions and 74 deletions.
diff --git a/deploy/templates/operator/manifests/crd.yaml b/deploy/templates/operator/manifests/crd.yaml
@@ -55,19 +55,23 @@ spec:
                         destination:
                           description: Address to export logs
                           type: string
+                        pathTemplate:
+                          description: Path constructed from log metadata for exporting
+                            logs
+                          type: string
                         rootPath:
-                          description: Root directory to store logs within external
-                            storage
+                          description: Deprecated; Root directory to store logs within
+                            external storage
                           type: string
                         shouldEncodeFileName:
                           description: Provide an option to convert '+' to '%2B' to
                             address issues in certain web environments where '+' is
                             misinterpreted
                           type: boolean
                         timeLayoutOfSubDirectory:
-                          description: An option(default `2006-01`) that sets the
-                            name of the sub-directory following `{Root path}` to a
-                            time-based layout
+                          description: Deprecated; An option(default `2006-01`) that
+                            sets the name of the sub-directory following `{Root path}`
+                            to a time-based layout
                           type: string
                       type: object
                     description:
@@ -209,12 +213,16 @@ spec:
                         destination:
                           description: S3 Address to export logs
                           type: string
+                        pathTemplate:
+                          description: Path constructed from log metadata for exporting
+                            logs
+                          type: string
                         region:
                           description: S3 region
                           type: string
                         rootPath:
-                          description: Root directory to store logs within external
-                            storage
+                          description: Deprecated; Root directory to store logs within
+                            external storage
                           type: string
                         secretKey:
                           description: S3 bucket secret key
@@ -230,9 +238,9 @@ spec:
                           description: Tags for objects to be stored
                           type: object
                         timeLayoutOfSubDirectory:
-                          description: An option(default `2006-01`) that sets the
-                            name of the sub-directory following `{Root path}` to a
-                            time-based layout
+                          description: Deprecated; An option(default `2006-01`) that
+                            sets the name of the sub-directory following `{Root path}`
+                            to a time-based layout
                           type: string
                       type: object
                   type: object

diff --git a/docs/design/log_sink.md b/docs/design/log_sink.md
@@ -70,8 +70,9 @@ spec:
       container: tc-container
     basicBucket:
       destination: http://{basicBucket endpoint}
-      rootPath: /
+      rootPath: / # deprecated
       timeLayoutOfSubDirectory: {time layout of sub-directory; default `2006-01`}
+      pathTemplate: "/path/{{TimeLayout \"20060102\"}}"
   - name: exclude-GET
     interval: 1h
     filter:
@@ -84,8 +85,9 @@ spec:
       namespace: log-test
     s3Bucket:
       destination: http://{s3 bucket endpoint}
-      rootPath: /
+      rootPath: / # deprecated
       timeLayoutOfSubDirectory: {time layout of sub-directory; default `2006-01`}
+      pathTemplate: "/path/{{TimeLayout \"20060102\"}}"
       bucketName: {s3 bucket name}
       accessKey: {s3 bucket access key}
       secretKey: {s3 bucket secret key}
@@ -105,7 +107,10 @@ spec:
 - `matcher` operates as a module within `store` and `exporter` operates as a sidecar for `store`
 - `exporter` periodically sends logs as a file to external storage. Since information about transfer volume or transfer failure is produced as Prometheus metrics, please refer to the [metrics](./metrics.md) guide for the metric contents
   - If a bucket has a directory structure, the directory can follow the rules below\
-    `{Root path}/{time layout of sub-directory}/{Namespace}/{LobsterSink name}/{Contents name}/{Pod}/{Container}/{Start time of log}_{End time of log}.log`
+    - AS-IS(deprecated): `{Root path}/{time layout of sub-directory}/{Namespace}/{LobsterSink name}/{Contents name}/{Pod}/{Container}/{Start time of log}_{End time of log}.log`
+    - TO-BE
+      - default: `/{time layout(2006-01)}/{Namespace}/{LobsterSink name}/{Contents name}/{Pod}/{Container}/{Start time of log}_{End time of log}.log`
+      - Using PathTemplate: `/{PathTemplate}/{Start time of log}_{End time of log}.log` (Please refer to the `PathTemplate` section below for more details.)
 - `time layout of sub-directory` is an option(default `2006-01`) that sets the name of the sub-directory following `{Root path}` to a time-based layout \
    Logs can be stored in a specific directory according to the rendering results of the layout, such as time or date
    - Here are some examples
@@ -134,4 +139,57 @@ spec:
      - https://pkg.go.dev/time#example-Time.Format
      - https://go.dev/src/time/format.go
 
-![overview_multi-cluster-with-logsink](../images/overview_multi-cluster-with-logsink.png)
+![overview_multi-cluster-with-logsink](../images/overview_multi-cluster-with-logsink.png)
+
+
+### PathTemplate
+
+- As the exported logs are transformed into various formats, the requirements for log paths have also increased
+- With the `PathTemplate` based on Go templates, you can freely structure fields using the template fields described below
+
+#### Template Fields
+- Reserved keywords are provided to allow flexible directory composition in the path
+- Each field can be used multiple times
+- If this feature is not used, the default path structure is applied
+  - `/{time layout(2006-01)}/{Namespace}/{LobsterSink name}/{Contents name}/{Pod}/{Container}/{Start time of log}_{End time of log}.log`
+
+Field | Description
+--- | ---
+`{{TimeLayout "2006-01-02"}}` | Time based directory layout 
+`{{.Cluster}}` | Cluster name
+`{{.Namespace}}` | Log source namespace name
+`{{.SinkName}}` | Lobster sink name
+`{{.RuleName}}` | Rule name in Lobster sink
+`{{.Pod}}` | Pod name
+`{{.Container}}` | Container name or `__emptydir__`
+`{{.SourceType}}` | `stdstream` or `emptydir`
+`{{.SourcePath}}` | A file path within an emptyDir volume
+
+#### Example
+
+```
+spec:
+  logExportRules:
+  - basicBucket:
+      pathTemplate: /pathtest/{{TimeLayout "20060102"}}/{{.Container}}
+  ...
+```
+
+PathTemplate | Exported file
+--- | ---
+`/lobster` | /lobster/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log 
+`/{{TimeLayout "2006-01"}}` | /2025-01/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log 
+`/{{.Pod}}` | /loggen-123/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log 
+`/{{.SourcePath}}/{{.Pod}}` | /renamed_namespaceA_test.log/loggen-123/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log 
+`/lobster/{{TimeLayout "2006-01"}}/123/{{.SourcePath}}` | /lobster/2025-01/123/renamed_namespaceA_test.log/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log 
+`/{{TimeLayout "2006-01"}}/{{TimeLayout "2006-01-02"}}` | /2025-01/2025-01-06/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log 
+
+#### Cautions
+
+- Customization introduces potential risks of conflicts due to insufficient delimiters
+- For example, with configurations like `/{{.Pod}}`, where delimiters are insufficient, \
+  conflicts may arise when exporting logs from multiple containers, each having the same log timestamp
+  - container A - `2025-01-06T18:20:24%252B09:00_2025-01-06T18:21:23%252B09:00.log`
+  - container B - `2025-01-06T18:20:24%252B09:00_2025-01-06T18:21:23%252B09:00.log`
+
+
diff --git a/pkg/docs/operator/docs.go b/pkg/docs/operator/docs.go
@@ -215,16 +215,20 @@ const docTemplate = `{
                     "description": "Address to export logs",
                     "type": "string"
                 },
+                "pathTemplate": {
+                    "description": "Path constructed from log metadata for exporting logs",
+                    "type": "string"
+                },
                 "rootPath": {
-                    "description": "Root directory to store logs within external storage",
+                    "description": "Deprecated; Root directory to store logs within external storage",
                     "type": "string"
                 },
                 "shouldEncodeFileName": {
                     "description": "Provide an option to convert '+' to '%2B' to address issues in certain web environments where '+' is misinterpreted",
                     "type": "boolean"
                 },
                 "timeLayoutOfSubDirectory": {
-                    "description": "An option(default ` + "`" + `2006-01` + "`" + `) that sets the name of the sub-directory following ` + "`" + `{Root path}` + "`" + ` to a time-based layout",
+                    "description": "Deprecated; An option(default ` + "`" + `2006-01` + "`" + `) that sets the name of the sub-directory following ` + "`" + `{Root path}` + "`" + ` to a time-based layout",
                     "type": "string",
                     "default": "2006-01"
                 }
@@ -422,12 +426,16 @@ const docTemplate = `{
                     "description": "S3 Address to export logs",
                     "type": "string"
                 },
+                "pathTemplate": {
+                    "description": "Path constructed from log metadata for exporting logs",
+                    "type": "string"
+                },
                 "region": {
                     "description": "S3 region",
                     "type": "string"
                 },
                 "rootPath": {
-                    "description": "Root directory to store logs within external storage",
+                    "description": "Deprecated; Root directory to store logs within external storage",
                     "type": "string"
                 },
                 "secretKey": {
@@ -447,7 +455,7 @@ const docTemplate = `{
                     ]
                 },
                 "timeLayoutOfSubDirectory": {
-                    "description": "An option(default ` + "`" + `2006-01` + "`" + `) that sets the name of the sub-directory following ` + "`" + `{Root path}` + "`" + ` to a time-based layout",
+                    "description": "Deprecated; An option(default ` + "`" + `2006-01` + "`" + `) that sets the name of the sub-directory following ` + "`" + `{Root path}` + "`" + ` to a time-based layout",
                     "type": "string",
                     "default": "2006-01"
                 }

diff --git a/pkg/docs/operator/swagger.json b/pkg/docs/operator/swagger.json
@@ -207,16 +207,20 @@
                     "description": "Address to export logs",
                     "type": "string"
                 },
+                "pathTemplate": {
+                    "description": "Path constructed from log metadata for exporting logs",
+                    "type": "string"
+                },
                 "rootPath": {
-                    "description": "Root directory to store logs within external storage",
+                    "description": "Deprecated; Root directory to store logs within external storage",
                     "type": "string"
                 },
                 "shouldEncodeFileName": {
                     "description": "Provide an option to convert '+' to '%2B' to address issues in certain web environments where '+' is misinterpreted",
                     "type": "boolean"
                 },
                 "timeLayoutOfSubDirectory": {
-                    "description": "An option(default `2006-01`) that sets the name of the sub-directory following `{Root path}` to a time-based layout",
+                    "description": "Deprecated; An option(default `2006-01`) that sets the name of the sub-directory following `{Root path}` to a time-based layout",
                     "type": "string",
                     "default": "2006-01"
                 }
@@ -414,12 +418,16 @@
                     "description": "S3 Address to export logs",
                     "type": "string"
                 },
+                "pathTemplate": {
+                    "description": "Path constructed from log metadata for exporting logs",
+                    "type": "string"
+                },
                 "region": {
                     "description": "S3 region",
                     "type": "string"
                 },
                 "rootPath": {
-                    "description": "Root directory to store logs within external storage",
+                    "description": "Deprecated; Root directory to store logs within external storage",
                     "type": "string"
                 },
                 "secretKey": {
@@ -439,7 +447,7 @@
                     ]
                 },
                 "timeLayoutOfSubDirectory": {
-                    "description": "An option(default `2006-01`) that sets the name of the sub-directory following `{Root path}` to a time-based layout",
+                    "description": "Deprecated; An option(default `2006-01`) that sets the name of the sub-directory following `{Root path}` to a time-based layout",
                     "type": "string",
                     "default": "2006-01"
                 }

diff --git a/pkg/docs/operator/swagger.yaml b/pkg/docs/operator/swagger.yaml
@@ -4,17 +4,20 @@ definitions:
       destination:
         description: Address to export logs
         type: string
+      pathTemplate:
+        description: Path constructed from log metadata for exporting logs
+        type: string
       rootPath:
-        description: Root directory to store logs within external storage
+        description: Deprecated; Root directory to store logs within external storage
         type: string
       shouldEncodeFileName:
         description: Provide an option to convert '+' to '%2B' to address issues in
           certain web environments where '+' is misinterpreted
         type: boolean
       timeLayoutOfSubDirectory:
         default: 2006-01
-        description: An option(default `2006-01`) that sets the name of the sub-directory
-          following `{Root path}` to a time-based layout
+        description: Deprecated; An option(default `2006-01`) that sets the name of
+          the sub-directory following `{Root path}` to a time-based layout
         type: string
     type: object
   v1.Filter:
@@ -142,11 +145,14 @@ definitions:
       destination:
         description: S3 Address to export logs
         type: string
+      pathTemplate:
+        description: Path constructed from log metadata for exporting logs
+        type: string
       region:
         description: S3 region
         type: string
       rootPath:
-        description: Root directory to store logs within external storage
+        description: Deprecated; Root directory to store logs within external storage
         type: string
       secretKey:
         description: S3 bucket secret key
@@ -161,8 +167,8 @@ definitions:
         description: Tags for objects to be stored
       timeLayoutOfSubDirectory:
         default: 2006-01
-        description: An option(default `2006-01`) that sets the name of the sub-directory
-          following `{Root path}` to a time-based layout
+        description: Deprecated; An option(default `2006-01`) that sets the name of
+          the sub-directory following `{Root path}` to a time-based layout
         type: string
     type: object
   v1.SASL:

diff --git a/pkg/lobster/client/client.go b/pkg/lobster/client/client.go
@@ -24,6 +24,7 @@ import (
 	"log"
 	"time"
 
+	"github.com/golang/glog"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -43,6 +44,7 @@ type Client struct {
 	hostName string
 	*kubernetes.Clientset
 	timeout time.Duration
+	cache   map[string]v1.Pod
 }
 
 func New() (Client, error) {
@@ -56,10 +58,10 @@ func New() (Client, error) {
 
 	clientset, err := kubernetes.NewForConfig(config)
 
-	return Client{*conf.HostName, clientset, timeout}, err
+	return Client{*conf.HostName, clientset, timeout, map[string]v1.Pod{}}, err
 }
 
-func (c Client) GetPods() (map[string]v1.Pod, error) {
+func (c *Client) GetPods() map[string]v1.Pod {
 	podMap := map[string]v1.Pod{}
 	podList := v1.PodList{}
 
@@ -71,16 +73,20 @@ func (c Client) GetPods() (map[string]v1.Pod, error) {
 		AbsPath(fmt.Sprintf("/api/v1/nodes/%s/proxy/pods", c.hostName)).
 		DoRaw(ctx)
 	if err != nil {
-		return podMap, err
+		glog.Warningf("using cached pod information: failed to make a request to the k8s API server or kubelet: %s", err.Error())
+		return c.cache
 	}
 
 	if err := json.Unmarshal(data, &podList); err != nil {
-		return podMap, err
+		glog.Warningf("using cached pod information: failed to unmarshal the pod list response: %s", err.Error())
+		return c.cache
 	}
 
 	for _, pod := range podList.Items {
 		podMap[string(pod.UID)] = pod
 	}
 
-	return podMap, nil
+	c.cache = podMap
+
+	return podMap
 }
diff --git a/pkg/lobster/distributor/distributor.go b/pkg/lobster/distributor/distributor.go
@@ -82,11 +82,7 @@ func (d *Distributor) Run(stopChan chan struct{}) {
 		for {
 			select {
 			case <-inspectTicker.C:
-				podMap, err := d.client.GetPods()
-				if err != nil {
-					glog.Error(err)
-					continue
-				}
+				podMap := d.client.GetPods()
 
 				if len(podMap) == 0 {
 					panic("no pods found")

diff --git a/pkg/lobster/sink/exporter/exporter.go b/pkg/lobster/sink/exporter/exporter.go
@@ -76,12 +76,7 @@ func (e *LogExporter) Run(stopChan chan struct{}) {
 			now := time.Now()
 			current := now.Truncate(time.Second)
 			newOrders := map[string]order.Order{}
-
-			podMap, err := e.client.GetPods()
-			if err != nil {
-				glog.Error(err)
-				continue
-			}
+			podMap := e.client.GetPods()
 
 			if len(podMap) == 0 {
 				panic("no pods found")
@@ -107,6 +102,7 @@ func (e *LogExporter) Run(stopChan chan struct{}) {
 
 				if err := uploader.Validate(); err != nil {
 					glog.Error(err)
+					metrics.AddSinkFailure(order.Request, order.SinkNamespace, order.SinkName, uploader.Type(), uploader.Name())
 					continue
 				}