Skip to content

Commit

Permalink
Add validator
Browse files Browse the repository at this point in the history
  • Loading branch information
@minhduc140583
minhduc140583 committed Aug 13, 2022
1 parent f76a3fa commit f30e8d3
Show file tree
Hide file tree
Showing 8 changed files with 1,247 additions and 31 deletions.
26 changes: 26 additions & 0 deletions import/error_checker.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package impt

import (
"context"
"fmt"
)

type ErrorChecker struct {
validate func(ctx context.Context, model interface{}) ([]ErrorMessage, error)
}

func NewErrorChecker(validate func(context.Context, interface{}) ([]ErrorMessage, error)) *ErrorChecker {
return &ErrorChecker{validate: validate}
}

func (v *ErrorChecker) Check(ctx context.Context, model interface{}) error {
errors, err := v.validate(ctx, model)
if err != nil {
return err
}
if errors != nil && len(errors) > 0 {
m := fmt.Sprintf("%s", errors)
return fmt.Errorf(m)
}
return nil
}
20 changes: 12 additions & 8 deletions import/file_reader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type FileReader struct {
func NewFileReader(buildFileName func() string) (*FileReader, error) {
return NewDelimiterFileReader(buildFileName, ',')
}
func NewDelimiterFileReader(buildFileName func() string, delimiter rune, opts... *encoding.Decoder) (*FileReader, error) {
func NewDelimiterFileReader(buildFileName func() string, delimiter rune, opts ...*encoding.Decoder) (*FileReader, error) {
var decoder *encoding.Decoder
if len(opts) > 0 && opts[0] != nil {
decoder = opts[0]
Expand All @@ -35,33 +35,35 @@ func NewDelimiterFileReader(buildFileName func() string, delimiter rune, opts...
return &fr, nil
}

func (fr *FileReader) Read(next func(lines []string, err error) error) error {
func (fr *FileReader) Read(next func(lines []string, err error, numLine int) error) error {
file, err := os.Open(fr.FileName)
if err != nil {
err = errors.New("cannot open file")
next(make([]string, 0), err)
next(make([]string, 0), err, 0)
return err
}

defer file.Close()

scanner := bufio.NewScanner(file)
i := 1
for scanner.Scan() {
if scanner.Text() != "" {
err := next([]string{scanner.Text()}, nil)
err := next([]string{scanner.Text()}, nil, i)
if err != nil {
return err
}
}
i++
}
next([]string{}, io.EOF)
next([]string{}, io.EOF, i)
return nil
}

func (fr *FileReader) ReadDelimiterFile(next func(lines []string, err error) error) error {
func (fr *FileReader) ReadDelimiterFile(next func(lines []string, err error, numLine int) error) error {
file, err := os.Open(fr.FileName)
if err != nil {
next(make([]string, 0), err)
next(make([]string, 0), err, 0)
}
var r *csv.Reader
if fr.Decoder != nil {
Expand All @@ -75,15 +77,17 @@ func (fr *FileReader) ReadDelimiterFile(next func(lines []string, err error) err
}

defer file.Close()
i := 1
for {
record, err := r.Read()
err2 := next(record, err)
err2 := next(record, err, i)
if err2 != nil {
return err2
}
if err == io.EOF {
break
}
i++
}
return err
}
171 changes: 148 additions & 23 deletions import/importer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,66 +2,191 @@ package impt

import (
"context"
"database/sql"
"fmt"
"io"
"reflect"
)

func NewImportRepository(db *sql.DB, modelType reflect.Type,
type ErrorMessage struct {
Field string `yaml:"field" mapstructure:"field" json:"field,omitempty" gorm:"column:field" bson:"field,omitempty" dynamodbav:"field,omitempty" firestore:"field,omitempty"`
Code string `yaml:"code" mapstructure:"code" json:"code,omitempty" gorm:"column:code" bson:"code,omitempty" dynamodbav:"code,omitempty" firestore:"code,omitempty"`
Param string `yaml:"param" mapstructure:"param" json:"param,omitempty" gorm:"column:param" bson:"param,omitempty" dynamodbav:"param,omitempty" firestore:"param,omitempty"`
Message string `yaml:"message" mapstructure:"message" json:"message,omitempty" gorm:"column:message" bson:"message,omitempty" dynamodbav:"message,omitempty" firestore:"message,omitempty"`
}

type ErrorHandler struct {
HandleError func(ctx context.Context, format string, fields map[string]interface{})
FileName string
LineNumber string
Map *map[string]interface{}
}

func NewErrorHandler(logger func(ctx context.Context, format string, fields map[string]interface{}), fileName string, lineNumber string, mp *map[string]interface{}) *ErrorHandler {
if len(fileName) <= 0 {
fileName = "filename"
}
if len(lineNumber) <= 0 {
lineNumber = "lineNumber"
}
return &ErrorHandler{
HandleError: logger,
FileName: fileName,
LineNumber: lineNumber,
Map: mp,
}
}

func (e *ErrorHandler) HandlerError(ctx context.Context, rs interface{}, err []ErrorMessage, i int, fileName string) {
var ext = make(map[string]interface{})
if e.Map != nil {
ext = *e.Map
}
if len(e.FileName) > 0 && len(e.LineNumber) > 0 {
if len(fileName) > 0 {
ext[e.FileName] = fileName
}
if i > 0 {
ext[e.LineNumber] = i
}
e.HandleError(ctx, fmt.Sprintf("Message is invalid: %+v . Error: %+v", rs, err), ext)
} else if len(e.FileName) > 0 {
if len(fileName) > 0 {
ext[e.FileName] = fileName
}
e.HandleError(ctx, fmt.Sprintf("Message is invalid: %+v . Error: %+v line: %d", rs, err, i), ext)
} else if len(e.LineNumber) > 0 {
if i > 0 {
ext[e.LineNumber] = i
}
e.HandleError(ctx, fmt.Sprintf("Message is invalid: %+v . Error: %+v filename:%s", rs, err, fileName), ext)
} else {
e.HandleError(ctx, fmt.Sprintf("Message is invalid: %+v . Error: %+v filename:%s line: %d", rs, err, fileName, i), ext)
}
}

func (e *ErrorHandler) HandlerException(ctx context.Context, rs interface{}, err error, i int, fileName string) {
var ext = make(map[string]interface{})
if e.Map != nil {
ext = *e.Map
}
if len(e.FileName) > 0 && len(e.LineNumber) > 0 {
if len(fileName) > 0 {
ext[e.FileName] = fileName
}
if i > 0 {
ext[e.LineNumber] = i
}
e.HandleError(ctx, fmt.Sprintf("Error to write: %+v . Error: %+v", rs, err), ext)
} else if len(e.FileName) > 0 {
if len(fileName) > 0 {
ext[e.FileName] = fileName
}
e.HandleError(ctx, fmt.Sprintf("Error to write: %+v . Error: %+v line: %d", rs, err, i), ext)
} else if len(e.LineNumber) > 0 {
if i > 0 {
ext[e.LineNumber] = i
}
e.HandleError(ctx, fmt.Sprintf("Error to write: %+v . Error: %+v filename:%s", rs, err, fileName), ext)
} else {
e.HandleError(ctx, fmt.Sprintf("Error to write: %+v . Error: %v filename: %s line: %d", rs, err, fileName, i), ext)
}
}

func NewImportRepository(modelType reflect.Type,
transform func(ctx context.Context, lines []string) (interface{}, error),
write func(ctx context.Context, data interface{}, endLineFlag bool) error,
read func(next func(lines []string, err error) error) error,
read func(next func(lines []string, err error, numLine int) error) error,
handleException func(ctx context.Context, rs interface{}, err error, i int, fileName string),
validate func(ctx context.Context, model interface{}) ([]ErrorMessage, error),
logError func(ctx context.Context, rs interface{}, err []ErrorMessage, i int, fileName string),
opt ...string,
) *Importer {
return NewImporter(db, modelType, transform, write, read)
return NewImporter(modelType, transform, write, read, handleException, validate, logError, opt...)
}
func NewImportAdapter(db *sql.DB, modelType reflect.Type,
func NewImportAdapter(modelType reflect.Type,
transform func(ctx context.Context, lines []string) (interface{}, error),
write func(ctx context.Context, data interface{}, endLineFlag bool) error,
read func(next func(lines []string, err error) error) error,
read func(next func(lines []string, err error, numLine int) error) error,
handleException func(ctx context.Context, rs interface{}, err error, i int, fileName string),
validate func(ctx context.Context, model interface{}) ([]ErrorMessage, error),
logError func(ctx context.Context, rs interface{}, err []ErrorMessage, i int, fileName string),
opt ...string,
) *Importer {
return NewImporter(db, modelType, transform, write, read)
return NewImporter(modelType, transform, write, read, handleException, validate, logError, opt...)
}
func NewImportService(db *sql.DB, modelType reflect.Type,
func NewImportService(modelType reflect.Type,
transform func(ctx context.Context, lines []string) (interface{}, error),
write func(ctx context.Context, data interface{}, endLineFlag bool) error,
read func(next func(lines []string, err error) error) error,
read func(next func(lines []string, err error, numLine int) error) error,
handleException func(ctx context.Context, rs interface{}, err error, i int, fileName string),
validate func(ctx context.Context, model interface{}) ([]ErrorMessage, error),
logError func(ctx context.Context, rs interface{}, err []ErrorMessage, i int, fileName string),
opt ...string,
) *Importer {
return NewImporter(db, modelType, transform, write, read)
return NewImporter(modelType, transform, write, read, handleException, validate, logError, opt...)
}
func NewImporter(db *sql.DB, modelType reflect.Type,
func NewImporter(modelType reflect.Type,
transform func(ctx context.Context, lines []string) (interface{}, error),
write func(ctx context.Context, data interface{}, endLineFlag bool) error,
read func(next func(lines []string, err error) error) error,
read func(next func(lines []string, err error, numLine int) error) error,
handleException func(ctx context.Context, rs interface{}, err error, i int, fileName string),
validate func(ctx context.Context, model interface{}) ([]ErrorMessage, error),
handleError func(ctx context.Context, rs interface{}, err []ErrorMessage, i int, fileName string),
opt ...string,
) *Importer {
return &Importer{DB: db, modelType: modelType, Transform: transform, Write: write, Read: read}
filename := ""
if len(opt) > 0 {
filename = opt[0]
}
return &Importer{modelType: modelType, Transform: transform, Write: write, Read: read, Validate: validate, HandleError: handleError, HandleException: handleException, Filename: filename}
}

type Importer struct {
DB *sql.DB
modelType reflect.Type
Transform func(ctx context.Context, lines []string) (interface{}, error)
Read func(next func(lines []string, err error) error) error
Write func(ctx context.Context, data interface{}, endLineFlag bool) error
modelType reflect.Type
Transform func(ctx context.Context, lines []string) (interface{}, error)
Read func(next func(lines []string, err error, numLine int) error) error
Write func(ctx context.Context, data interface{}, endLineFlag bool) error
Validate func(ctx context.Context, model interface{}) ([]ErrorMessage, error)
HandleError func(ctx context.Context, rs interface{}, err []ErrorMessage, i int, fileName string)
HandleException func(ctx context.Context, rs interface{}, err error, i int, fileName string)
Filename string
}

func (s *Importer) Import(ctx context.Context) (err error) {
err = s.Read(func(lines []string, err error) error {
func (s *Importer) Import(ctx context.Context) (total int, success int, err error) {
err = s.Read(func(lines []string, err error, numLine int) error {
if err == io.EOF {
err = s.Write(ctx, nil, true)
return nil
}
total++
itemStruct, err := s.Transform(ctx, lines)
if err != nil {
return err
}
if s.Validate != nil {
errs, err := s.Validate(ctx, itemStruct)
if err != nil {
return err
}
if len(errs) > 0 {
s.HandleError(ctx, itemStruct, errs, numLine, s.Filename)
return nil
}
}
err = s.Write(ctx, itemStruct, false)
if err != nil {
return err
if s.HandleException != nil {
s.HandleException(ctx, itemStruct, err, numLine, s.Filename)
return nil
} else {
return err
}
}
success++
return nil
})
if err != nil && err != io.EOF {
return err
return total, success, err
}
return nil
return total, success, nil
}
Loading

0 comments on commit f30e8d3

Please sign in to comment.