Skip to content

Commit

Permalink
Optimize performance for reader and add sql
Browse files Browse the repository at this point in the history
  • Loading branch information
minhduc140583 committed Jul 14, 2024
1 parent a757478 commit d0e3caf
Show file tree
Hide file tree
Showing 17 changed files with 1,905 additions and 187 deletions.
2 changes: 1 addition & 1 deletion formatter/delimiter_formatter.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,5 @@ func NewDelimiterFormatter[T any](opts ...string) (*DelimiterFormatter[T], error
}

func (f *DelimiterFormatter[T]) Format(ctx context.Context, model *T) string {
return writer.ToTextWithDelimiter(ctx, model, f.Delimiter, f.formatCols)
return writer.ToTextWithDelimiter(model, f.Delimiter, f.formatCols)
}
152 changes: 152 additions & 0 deletions reader/common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package reader

import (
"math/big"
"reflect"
"strconv"
"time"
)

const DateLayout string = "2006-01-02 15:04:05 +0700 +07"

var funcMap = map[string]func(f reflect.Value, line string, format string, scale int) error{
"string": HandleString,
"*string": HandleString,
"time.Time": HandleTime,
"*time.Time": HandleTime,
"bool": HandleBool,
"*bool": HandleBool,
"int": HandleInt,
"*int": HandleInt,
"int64": HandleInt,
"*int64": HandleInt64,
"int32": HandleInt32,
"*int32": HandleInt32,
"big.Int": HandleBigInt,
"*big.Int": HandleBigInt,
"float64": HandleFloat64,
"*float64": HandleFloat64,
"big.Float": HandleBigFloat,
"*big.Float": HandleBigFloat,
}

func HandleUnknown(f reflect.Value, line string, format string, scale int) error {
return nil
}
func HandleString(f reflect.Value, line string, format string, scale int) error {
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&line))
} else {
f.SetString(line)
}
return nil
}
func HandleTime(f reflect.Value, line string, format string, scale int) error {
var fieldDate time.Time
var err error
if len(format) > 0 {
fieldDate, err = time.Parse(format, line)
} else {
fieldDate, err = time.Parse(DateLayout, line)
}
if err != nil {
return err
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&fieldDate))
} else {
f.Set(reflect.ValueOf(fieldDate))
}
return nil
}
func HandleFloat64(f reflect.Value, line string, format string, scale int) error {
floatValue, err := strconv.ParseFloat(line, 64)
if err != nil {
return err
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&floatValue))
} else {
f.SetFloat(floatValue)
}
return nil
}
func HandleInt32(f reflect.Value, line string, format string, scale int) error {
value, err := strconv.Atoi(line)
if err != nil {
return err
}
var i int32 = int32(value)
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&i))
} else {
f.Set(reflect.ValueOf(i))
}
return nil
}
func HandleInt64(f reflect.Value, line string, format string, scale int) error {
value, err := strconv.ParseInt(line, 10, 64)
if err != nil {
return err
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&value))
} else {
f.SetInt(value)
}
return nil
}
func HandleInt(f reflect.Value, line string, format string, scale int) error {
value, err := strconv.Atoi(line)
if err != nil {
return err
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&value))
} else {
f.Set(reflect.ValueOf(value))
}
return nil
}
func HandleBool(f reflect.Value, line string, format string, scale int) error {
boolValue, err := strconv.ParseBool(line)
if err != nil {
return err
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&boolValue))
} else {
f.SetBool(boolValue)
}
return nil
}
func HandleBigFloat(f reflect.Value, line string, format string, scale int) error {
bf := new(big.Float)
if bfv, ok := bf.SetString(line); ok {
if scale >= 0 && bfv != nil {
k := Round(*bf, scale)
bf = &k
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(bfv))
} else {
if bfv != nil {
f.Set(reflect.ValueOf(*bfv))
}
}
}
return nil
}
func HandleBigInt(f reflect.Value, line string, format string, scale int) error {
bf := new(big.Int)
if bfv, oki := bf.SetString(line, 10); oki {
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(bfv))
} else {
if bfv != nil {
f.Set(reflect.ValueOf(*bfv))
}
}
}
return nil
}
108 changes: 15 additions & 93 deletions reader/delimiter_transformer.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,9 @@ package reader
import (
"context"
"errors"
"math/big"
"reflect"
"strconv"
"strings"
"time"
)

const (
DateLayout string = "2006-01-02 15:04:05 +0700 +07"
)

func GetIndexesByTag(modelType reflect.Type, tagName string) (map[int]Delimiter, error) {
Expand Down Expand Up @@ -44,6 +38,13 @@ func GetIndexesByTag(modelType reflect.Type, tagName string) (map[int]Delimiter,
}
v.Format = tagValue
}
v.TypeName = field.Type.String()
fn, ok := funcMap[v.TypeName]
if ok {
v.Handle = fn
} else {
v.Handle = HandleUnknown
}
ma[i] = v
}
}
Expand All @@ -70,8 +71,10 @@ type DelimiterFormatter struct {
}

type Delimiter struct {
Format string
Scale int
TypeName string
Format string
Scale int
Handle func(f reflect.Value, line string, format string, scale int) error
}

func (f DelimiterFormatter) ToStruct(ctx context.Context, lineStr string, res interface{}) error {
Expand All @@ -86,99 +89,18 @@ func Min(n1 int, n2 int) int {
return n2
}
func ScanLine(lines []string, record interface{}, formatCols map[int]Delimiter) error {
modelType := reflect.TypeOf(record).Elem()
s := reflect.Indirect(reflect.ValueOf(record))
numFields := s.NumField()
l := len(formatCols)
le := Min(numFields, l)
for i := 0; i < le; i++ {
field := modelType.Field(i)
typef := field.Type.String()
line := lines[i]
f := s.Field(i)
if f.CanSet() {
switch typef {
case "string", "*string":
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&line))
} else {
f.SetString(line)
}
case "time.Time", "*time.Time":
if format, ok := formatCols[i]; ok {
var fieldDate time.Time
var err error
if len(format.Format) > 0 {
fieldDate, err = time.Parse(format.Format, line)
} else {
fieldDate, err = time.Parse(DateLayout, line)
}
if err != nil {
return err
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&fieldDate))
} else {
f.Set(reflect.ValueOf(fieldDate))
}
}
case "float64", "*float64":
floatValue, _ := strconv.ParseFloat(line, 64)
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&floatValue))
} else {
f.SetFloat(floatValue)
}
case "int64", "*int64":
value, _ := strconv.ParseInt(line, 10, 64)
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&value))
} else {
f.SetInt(value)
}
case "int", "*int":
value, _ := strconv.Atoi(line)
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&value))
} else {
f.Set(reflect.ValueOf(value))
}
case "bool", "*bool":
boolValue, _ := strconv.ParseBool(line)
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(&boolValue))
} else {
f.SetBool(boolValue)
}
case "big.Float", "*big.Float":
if formatf, ok := formatCols[i]; ok {
bf := new(big.Float)
if bfv, ok := bf.SetString(line); ok {
if formatf.Scale >= 0 && bfv != nil {
k := Round(*bf, formatf.Scale)
bf = &k
}
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(bfv))
} else {
if bfv != nil {
f.Set(reflect.ValueOf(*bfv))
}
}
}
}
case "big.Int", "*big.Int":
if _, ok := formatCols[i]; ok {
bf := new(big.Int)
if bfv, oki := bf.SetString(line, 10); oki {
if f.Kind() == reflect.Ptr {
f.Set(reflect.ValueOf(bfv))
} else {
if bfv != nil {
f.Set(reflect.ValueOf(*bfv))
}
}
}
if format, ok := formatCols[i]; ok {
err := format.Handle(f, line, format.Format, format.Scale)
if err != nil {
return err
}
}
}
Expand Down
Loading

0 comments on commit d0e3caf

Please sign in to comment.