Skip to content

Commit

Permalink
fix docs, field name and template name
Browse files Browse the repository at this point in the history
  • Loading branch information
@sharkpc138
sharkpc138 committed Jan 8, 2025
1 parent 1c5c437 commit e53b379
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 24 deletions.
17 changes: 9 additions & 8 deletions docs/design/log_sink.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ spec:
destination: http://{basicBucket endpoint}
rootPath: / # deprecated
timeLayoutOfSubDirectory: {time layout of sub-directory; default `2006-01`}
pathTemplate: "/path/{{TimeFunc \"20060102\"}}"
pathTemplate: "/path/{{TimeLayout \"20060102\"}}"
- name: exclude-GET
interval: 1h
filter:
Expand All @@ -87,7 +87,7 @@ spec:
destination: http://{s3 bucket endpoint}
rootPath: / # deprecated
timeLayoutOfSubDirectory: {time layout of sub-directory; default `2006-01`}
pathTemplate: "/path/{{TimeFunc \"20060102\"}}"
pathTemplate: "/path/{{TimeLayout \"20060102\"}}"
bucketName: {s3 bucket name}
accessKey: {s3 bucket access key}
secretKey: {s3 bucket secret key}
Expand Down Expand Up @@ -155,9 +155,10 @@ spec:
Field | Description
--- | ---
`{{TimeFunc "2006-01-02"}}` | Time based directory layout
`{{TimeLayout "2006-01-02"}}` | Time based directory layout
`{{.Namespace}}` | Log source namespace name
`{{.SinkName}}` | Lobster sink name
`{{.RuleName}}` | Rule name in Lobster sink
`{{.Pod}}` | Pod name
`{{.Container}}` | Container name or `__emptydir__`
`{{.SourceType}}` | `stdstream` or `emptydir`
Expand All @@ -170,18 +171,18 @@ spec:
logExportRules:
- basicBucket:
destination: http://a-dev.nubes.sto.navercorp.com:8000/v1/n3r-logging
pathTemplate: /pathtest/{{TimeFunc "20060102"}}/{{.Container}}
pathTemplate: /pathtest/{{TimeLayout "20060102"}}/{{.Container}}
...
```
PathTemplate | Exported file
--- | ---
`/lobster` | /lobster/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{TimeFunc "2006-01"}}` | /2025-01/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{TimeLayout "2006-01"}}` | /2025-01/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{.Pod}}` | /loggen-123/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{.SourcePath}}` | /renamed_namespaceA_test.log/loggen-123/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/lobster/{{TimeFunc "2006-01"}}/123/{{.SourcePath}}` | /lobster/2025-01/123/renamed_namespaceA_test.log/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{TimeFunc "2006-01"}}/{{Time "2006-01-02"}}` | /2025-01-06/2025-01/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{.SourcePath}}/{{.Pod}}` | /renamed_namespaceA_test.log/loggen-123/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/lobster/{{TimeLayout "2006-01"}}/123/{{.SourcePath}}` | /lobster/2025-01/123/renamed_namespaceA_test.log/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
`/{{TimeLayout "2006-01"}}/{{TimeLayout "2006-01-02"}}` | /2025-01/2025-01-06/2025-01-06T14:17:15%2B09:00_2025-01-06T14:17:17%2B09:00.log
#### Cautions
Expand Down
3 changes: 2 additions & 1 deletion pkg/lobster/sink/exporter/uploader/basic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,8 @@ func (b BasicUploader) templateDir(chunk model.Chunk, date time.Time) (string, e
b.Order.LogExportRule.BasicBucket.PathTemplate,
template.PathElement{
Namespace: chunk.Namespace,
SinkName: b.Order.LogExportRule.Name,
SinkName: b.Order.SinkName,
RuleName: b.Order.LogExportRule.Name,
Pod: chunk.Pod,
Container: chunk.Container,
SourceType: chunk.Source.Type,
Expand Down
3 changes: 2 additions & 1 deletion pkg/lobster/sink/exporter/uploader/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ func (s S3Uploader) templateDir(chunk model.Chunk, date time.Time) (string, erro
s.Order.LogExportRule.S3Bucket.PathTemplate,
template.PathElement{
Namespace: chunk.Namespace,
SinkName: s.Order.LogExportRule.Name,
SinkName: s.Order.SinkName,
RuleName: s.Order.LogExportRule.Name,
Pod: chunk.Pod,
Container: chunk.Container,
SourceType: chunk.Source.Type,
Expand Down
8 changes: 5 additions & 3 deletions pkg/operator/api/v1/template/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package template
import (
"bytes"
"errors"
"fmt"
"net/url"
"strings"
"text/template"
Expand All @@ -36,14 +37,15 @@ func init() {
type PathElement struct {
Namespace string
SinkName string
RuleName string
Pod string
Container string
SourceType string
SourcePath string
TimeInput time.Time
}

func (d PathElement) Time(layout string) string {
func (d PathElement) TimeFormat(layout string) string {
return d.TimeInput.Format(layout)
}

Expand All @@ -52,7 +54,7 @@ func ValidateTemplateString(templateStr string) error {
return errors.New("mismatch between '{{' and '}}'")
}

tmpl, err := getTemplate(templateStr, PathElement{}).Parse(templateStr)
tmpl, err := getTemplate(fmt.Sprintf("validate_%s", templateStr), PathElement{}).Parse(templateStr)
if err != nil {
return err
}
Expand Down Expand Up @@ -84,7 +86,7 @@ func getTemplate(templateStr string, elem PathElement) *template.Template {
v, ok := templateCache.Get(templateStr)
if !ok {
newTmpl := template.New(templateStr).Funcs(template.FuncMap{
"TimeFunc": elem.Time,
"TimeLayout": elem.TimeFormat,
})
templateCache.Add(templateStr, newTmpl)

Expand Down
22 changes: 11 additions & 11 deletions pkg/operator/api/v1/template/template_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@ import (

func TestGeneratePath(t *testing.T) {
templates := map[string]string{
"/my": "/my",
"/{{TimeFunc \"2006-01\"}}": "/2025-01",
"/{{.Pod}}": "/loggen-pod",
"/{{.Container}}/{{.Pod}}": "/container1/loggen-pod",
"/{{.SourcePath}}": "/renamed_namespaceA_test.log",
"/my/{{TimeFunc \"2006-01\"}}/123/{{.SourcePath}}": "/my/2025-01/123/renamed_namespaceA_test.log",
"/{{.Pod}}-{{.Container}}/{{.Pod}}": "/loggen-pod-container1/loggen-pod",
"/{{TimeFunc \"2006-01\"}}/{{TimeFunc \"2006-01-02\"}}/{{TimeFunc \"2006-01-02_15\"}}": "/2025-01/2025-01-06/2025-01-06_14",
"/my": "/my",
"/{{TimeLayout \"2006-01\"}}": "/2025-01",
"/{{.Pod}}": "/loggen-pod",
"/{{.Container}}/{{.Pod}}": "/container1/loggen-pod",
"/{{.SourcePath}}": "/renamed_namespaceA_test.log",
"/my/{{TimeLayout \"2006-01\"}}/123/{{.SourcePath}}": "/my/2025-01/123/renamed_namespaceA_test.log",
"/{{.Pod}}-{{.Container}}/{{.Pod}}": "/loggen-pod-container1/loggen-pod",
"/{{TimeLayout \"2006-01\"}}/{{TimeLayout \"2006-01-02\"}}/{{TimeLayout \"2006-01-02_15\"}}": "/2025-01/2025-01-06/2025-01-06_14",
}

timeInput := time.Date(2025, 1, 6, 14, 17, 15, 0, time.UTC)
Expand Down Expand Up @@ -61,9 +61,9 @@ func TestGeneratePath(t *testing.T) {

func TestInvalidPath(t *testing.T) {
templates := []string{
"/{.SourcePath}}", // invalid brace
"/{{.TimeFunc \"2006-01\"}}", // invalid function usage
"/{{TimeFunc}}", // invalid function usage
"/{.SourcePath}}", // invalid brace
"/{{.TimeLayout \"2006-01\"}}", // invalid function usage
"/{{TimeLayout}}", // invalid function usage
}

for _, tmpl := range templates {
Expand Down

0 comments on commit e53b379

Please sign in to comment.