diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..48be106 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,25 @@ +name: Go +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: [ '1.15.x', 'stable' ] + defaults: + run: + working-directory: ./go + steps: + - uses: actions/checkout@v4 + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + cache-dependency-path: ./go/go.sum + - name: Display Go version + run: go version + - name: Download dependencies + run: go mod download + - name: Test with Go + run: go test -v -cover ./... diff --git a/go/Makefile b/go/Makefile index 26cbb27..5ba4c30 100644 --- a/go/Makefile +++ b/go/Makefile @@ -18,7 +18,10 @@ BUILD_ENV=GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 bin/%: $(wildcard *.go) $(BUILD_ENV) go build -v -o $@ -ldflags "-w -s" +test: + go test -cover ./... + clean: rm -r bin -.PHONY: all clean \ No newline at end of file +.PHONY: all clean diff --git a/go/collector_test.go b/go/collector_test.go new file mode 100644 index 0000000..dbcae31 --- /dev/null +++ b/go/collector_test.go @@ -0,0 +1,261 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" +) + +type filedata struct { + path string + content []byte +} + +func (f filedata) String() string { + return fmt.Sprintf("%s: %s", f.path, PrettyPrintBytes(f.content)) +} + +func PrettyPrintBytes(b []byte) string { + const maxLen = 40 + var curLen = len(b) + ellipsis := "" + if len(b) > maxLen { + ellipsis = "..." + curLen = maxLen + } + return fmt.Sprintf("%[1]q%[2]s (%[1]v%[2]s)", b[:curLen], ellipsis) +} + +func getFileName(file *multipart.Part) string { + var fileName string + disposition := file.Header.Get("Content-Disposition") + _, dispositionParams, err := mime.ParseMediaType(disposition) + if err == nil { + fileName = dispositionParams["filename"] + } + return fileName +} +func collectSendAndReceive(cc CollectorConfig, t *testing.T) ([]filedata, CollectionStatistics) { + receivedFiles := make([]filedata, 0) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reader, err := r.MultipartReader() + if err != nil { + t.Fatalf("No multipart form received") + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + part, err := reader.NextPart() + if err == io.EOF { + t.Fatalf("No file received") + http.Error(w, "No file received", http.StatusBadRequest) + return + } else if err != nil { + t.Fatalf("Invalid multipart form") + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if part.FormName() != "file" { + t.Fatalf("Received upload with form name other than file") + http.Error(w, "Please use \"file\" as name for your upload", http.StatusBadRequest) + return + } + receivedFile := filedata{path: getFileName(part)} + receivedFile.content, err = ioutil.ReadAll(part) + if err != nil { + t.Fatalf("Failed to read multipart file") + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + receivedFiles = append(receivedFiles, receivedFile) + + if err := json.NewEncoder(w).Encode(map[string]interface{}{"id": uuid.New().String()}); err != nil { + t.Fatalf("Failed to write JSON response") + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + })) + defer ts.Close() + + logger := log.New(os.Stdout, "", log.Ldate|log.Ltime) + cc.Server = ts.URL + cc.Threads = 1 + cc.Debug = true + c := NewCollector(cc, logger) + c.StartWorkers() + c.Collect() + c.Stop() + + return receivedFiles, *c.Statistics +} + +func TestUpload(t *testing.T) { + testStartTime := time.Now() + testRoot := "testdata" + + _ = os.Chtimes(filepath.Join(testRoot, "foo.txt"), time.Time{}, time.Now().Local()) // Touch + + testCases := []struct { + name string + cc CollectorConfig + expectedFilesFound []filedata + expectedStats CollectionStatistics + }{ + { + "with excludes", + CollectorConfig{ + RootPaths: []string{testRoot}, + ExcludeGlobs: []string{"**/sub?", "**/*.jpg"}, + }, + []filedata{ + {"foo.txt", []byte("foo\n")}, + }, + CollectionStatistics{ + uploadedFiles: 1, + skippedFiles: 1, + skippedDirectories: 2, + }, + }, + { + "with extensions", + CollectorConfig{ + RootPaths: []string{testRoot}, + FileExtensions: []string{".nfo"}, + }, + []filedata{ + {"sub2/bar.nfo", []byte("bar\n")}, + }, + CollectionStatistics{ + uploadedFiles: 1, + skippedFiles: 3, + }, + }, + { + "with magic", + CollectorConfig{ + RootPaths: []string{testRoot}, + MagicHeaders: [][]byte{{0xff, 0xd8, 0xff}}, + }, + []filedata{ + {"nextron250.jpg", func() []byte { b, _ := ioutil.ReadFile("testdata/nextron250.jpg"); return b }()}, + }, + CollectionStatistics{ + uploadedFiles: 1, + skippedFiles: 3, + }, + }, + { + "with max age", + CollectorConfig{ + RootPaths: []string{testRoot}, + ThresholdTime: testStartTime, + }, + []filedata{ + {"foo.txt", []byte("foo\n")}, + }, + CollectionStatistics{ + uploadedFiles: 1, + skippedFiles: 3, + }, + }, + { + "with max size", + CollectorConfig{ + RootPaths: []string{testRoot}, + FileExtensions: []string{".jpg"}, + MaxFileSize: 14670, + }, + []filedata{}, + CollectionStatistics{ + skippedFiles: 4, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + receivedFiles, stats := collectSendAndReceive(tc.cc, t) + + if len(receivedFiles) != len(tc.expectedFilesFound) { + t.Fatalf("Expected to receive %d files, but received %d", len(tc.expectedFilesFound), len(receivedFiles)) + } + if diff := cmp.Diff(tc.expectedStats, stats, cmp.Exporter(func(_ reflect.Type) bool { return true })); diff != "" { + t.Fatalf("Statistics mismatch: %s", diff) + } + for _, expected := range tc.expectedFilesFound { + t.Run(fmt.Sprintf("File %s", expected.path), func(t *testing.T) { + found := false + expectedPathRel := filepath.Join(testRoot, expected.path) + expectedPathAbs, _ := filepath.Abs(expectedPathRel) + for _, received := range receivedFiles { + if received.path == expectedPathAbs { + if !cmp.Equal(received.content, expected.content) { + t.Fatalf("Content mismatch for file %s.\nExpected: %s\nGot: %s", received.path, PrettyPrintBytes(expected.content), PrettyPrintBytes(received.content)) + } + found = true + break + } + } + if !found { + t.Fatalf("Expected file %s not found in: %s", expectedPathAbs, receivedFiles) + } + }) + } + }) + } +} + +func TestCollect(t *testing.T) { + testRoot := "testdata" + cc := CollectorConfig{ + RootPaths: []string{testRoot}, + ExcludeGlobs: []string{"**/sub1", "**/*.jpg"}, + } + expectedFilesFound := []string{ + "foo.txt", + "sub2/bar.nfo", + } + logger := log.New(os.Stdout, "", log.Ldate|log.Ltime) + c := NewCollector(cc, logger) + c.filesToUpload = make(chan infoWithPath) + + var listenerThreadClosed = make(chan struct{}) + var collectedFiles []infoWithPath + go func() { + for incFile := range c.filesToUpload { + collectedFiles = append(collectedFiles, incFile) + } + close(listenerThreadClosed) + }() + c.Collect() + close(c.filesToUpload) + <-listenerThreadClosed + + numFound := 0 + for _, collected := range collectedFiles { + for _, expected := range expectedFilesFound { + if collected.path == filepath.Join(testRoot, expected) { + numFound++ + } + } + } + if numFound != len(collectedFiles) { + t.Fatalf("Expected to collect %d files, but collected %d. Expected: %v. Collected: %v", len(expectedFilesFound), len(collectedFiles), expectedFilesFound, collectedFiles) + } + if numFound != len(expectedFilesFound) { + t.Fatalf("Expected to find %d files, but found %d. Expected: %v. Found: %v", len(expectedFilesFound), numFound, expectedFilesFound, collectedFiles) + } +} diff --git a/go/go.mod b/go/go.mod index cf6d05e..676d498 100644 --- a/go/go.mod +++ b/go/go.mod @@ -4,6 +4,8 @@ go 1.15 require ( github.com/bmatcuk/doublestar/v3 v3.0.0 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 github.com/spf13/pflag v1.0.5 gopkg.in/yaml.v3 v3.0.0 ) diff --git a/go/go.sum b/go/go.sum index 989c867..e46ab48 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,5 +1,9 @@ github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI= github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/go/testdata/foo.txt b/go/testdata/foo.txt new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/go/testdata/foo.txt @@ -0,0 +1 @@ +foo diff --git a/go/testdata/nextron250.jpg b/go/testdata/nextron250.jpg new file mode 100644 index 0000000..9f380a1 Binary files /dev/null and b/go/testdata/nextron250.jpg differ diff --git a/go/testdata/sub1/word.docx b/go/testdata/sub1/word.docx new file mode 100644 index 0000000..e69de29 diff --git a/go/testdata/sub2/bar.nfo b/go/testdata/sub2/bar.nfo new file mode 100644 index 0000000..5716ca5 --- /dev/null +++ b/go/testdata/sub2/bar.nfo @@ -0,0 +1 @@ +bar