diff --git a/.gitignore b/.gitignore index 45fbacd..c495f5c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ _test *.[568vq] [568vq].out +# IDE settings +.idea/ + *.cgo1.go *.cgo2.c _cgo_defun.c diff --git a/README.md b/README.md index 2504ffa..9202dcb 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,55 @@ docker run --rm \ -w $(pwd) \ plugins/gcs ``` + +#### Optional object folder setting + +The object folder can be specified as part of the `PLUGIN_TARGET` or as part of the `PLUGIN_FOLDER` setting. The value +of the `PLUGIN_FOLDER` setting will be appended to any value supplied as part of the `PLUGIN_TARGET`. + +```console +docker run --rm \ + -e PLUGIN_SOURCE="dist" \ + -e PLUGIN_TARGET="bucket/dir/" \ + -e PLUGIN_FOLDER="${DRONE_PULL_REQUEST}" \ + -e PLUGIN_IGNORE="bin/*" \ + -e PLUGIN_ACL="allUsers:READER,user@domain.com:OWNER" \ + -e PLUGIN_GZIP="js,css,html" \ + -e PLUGIN_CACHE_CONTROL="public,max-age=3600" \ + -e PLUGIN_METADATA='{"x-goog-meta-foo":"bar"}' \ + -v $(pwd):$(pwd) \ + -w $(pwd) \ + plugins/gcs +``` + +One potential use case for specifying the `PLUGIN_FOLDER` separately is when storing the value for `PLUGIN_TARGET` as a +drone secret while needing to generate a final object folder value during CI. The above example illustrates how it may +be possible to publish to `bucket/dir/DRONE_PULL_REQUEST`. + +Perhaps a more illustrative example: + +```yaml +kind: pipeline +name: upload prs + +trigger: + branch: + include: + - develop + event: + - pull_request + +steps: + - name: upload to cloud storage prs bucket + image: plugins/gcs + settings: + source: build + target: + from_secret: prs_bucket + folder: "prs/${DRONE_PULL_REQUEST}" + gzip: js,css,html + token: + from_secret: prs_token +``` + +The above drone configuration file is not meant to be complete. diff --git a/go.mod b/go.mod index d37f2a1..efc5968 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,19 @@ go 1.15 require ( cloud.google.com/go v0.21.0 - github.com/golang/protobuf v1.0.0 - github.com/googleapis/gax-go v2.0.0+incompatible + github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect + github.com/golang/protobuf v1.0.0 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/googleapis/gax-go v2.0.0+incompatible // indirect github.com/pkg/errors v0.8.0 github.com/urfave/cli v1.20.0 - go.opencensus.io v0.8.0 + go.opencensus.io v0.8.0 // indirect golang.org/x/net v0.0.0-20180415214307-500e7a4f953d golang.org/x/oauth2 v0.0.0-20180402223937-921ae394b943 - golang.org/x/text v0.3.0 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/text v0.3.0 // indirect google.golang.org/api v0.0.0-20180413000347-fca24fcb4112 - google.golang.org/appengine v1.0.0 - google.golang.org/genproto v0.0.0-20180413175816-7fd901a49ba6 - google.golang.org/grpc v1.11.3 + google.golang.org/appengine v1.0.0 // indirect + google.golang.org/genproto v0.0.0-20180413175816-7fd901a49ba6 // indirect + google.golang.org/grpc v1.11.3 // indirect ) diff --git a/go.sum b/go.sum index 8e95158..446a7a8 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ cloud.google.com/go v0.21.0 h1:083TLU7hqb1CzKeMhzDsCTP4uXkBGr6nnWuaHdVNBCI= cloud.google.com/go v0.21.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/protobuf v1.0.0 h1:lsek0oXi8iFE9L+EXARyHIjU5rlWIhhTkjDz3vHhWWQ= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= @@ -14,10 +18,15 @@ golang.org/x/net v0.0.0-20180415214307-500e7a4f953d h1:YbPBOQdSo5fFDiZgBDcltE4Xj golang.org/x/net v0.0.0-20180415214307-500e7a4f953d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20180402223937-921ae394b943 h1:hE+k6oRG1aru6/y8A0HI01Zqp65EZcUcKToTkbPsFFM= golang.org/x/oauth2 v0.0.0-20180402223937-921ae394b943/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180413000347-fca24fcb4112 h1:ZwIBv243B9oUEb2HVMTQs//+WVhsL5hsUoCAu1ydobs= google.golang.org/api v0.0.0-20180413000347-fca24fcb4112/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/appengine v1.0.0 h1:dN4LljjBKVChsv0XCSI+zbyzdqrkEwX5LQFUMRSGqOc= google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180413175816-7fd901a49ba6 h1:VrRtqEIrO5wUzNwL/A2WTNUtDuAtvb3KPK3OrUriLqI= google.golang.org/genproto v0.0.0-20180413175816-7fd901a49ba6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/main.go b/main.go index 46608ff..249cc15 100644 --- a/main.go +++ b/main.go @@ -52,9 +52,14 @@ func main() { }, cli.StringFlag{ Name: "target", - Usage: "destination to copy files to, including bucket name", + Usage: "destination bucket to copy files to, can include folder path", EnvVar: "PLUGIN_TARGET", }, + cli.StringFlag{ + Name: "folder", + Usage: "destination folder path to copy files to, will be appended to target", + EnvVar: "PLUGIN_FOLDER", + }, cli.StringSliceFlag{ Name: "gzip", Usage: `files with the specified extensions will be gzipped and uploaded with "gzip" Content-Encoding header`, @@ -84,6 +89,7 @@ func run(c *cli.Context) error { ACL: c.StringSlice("acl"), Source: c.String("source"), Target: c.String("target"), + Folder: c.String("folder"), Ignore: c.String("ignore"), Gzip: c.StringSlice("gzip"), CacheControl: c.String("cache-control"), diff --git a/plugin.go b/plugin.go index e4d7df7..8f2d5fc 100644 --- a/plugin.go +++ b/plugin.go @@ -30,9 +30,12 @@ type ( // Copies the files from the specified directory. Source string - // Destination to copy files to, including bucket name + // Destination bucket to copy files to, can include folder path Target string + // Destination folder path to copy files to, will be appended to target + Folder string + // Exclude files matching this pattern. Ignore string @@ -113,11 +116,11 @@ func (p *Plugin) Exec(client *storage.Client) error { rel, err := filepath.Rel(p.Config.Source, f) if err != nil { - res <- &result{f, err} + res <- &result{rel, err} return } - err = p.uploadFile(path.Join(p.Config.Target, rel), f) + err = p.uploadFile(f) res <- &result{rel, err} <-buf // free up @@ -146,9 +149,26 @@ func (p *Plugin) errorf(format string, args ...interface{}) { p.printf(format, args...) } -// uploadFile uploads the file to dst using global bucket. +// objectFolderPath returns a bucket object folder name based on plugin target and folder values. +// It is intended that the bucket name has already been stripped from the target. +func (p *Plugin) objectFolderPath() string { + return path.Join(p.Config.Target, p.Config.Folder) +} + +// objectFolderPathForFile returns a bucket object name including a resolved object folder path. +func (p *Plugin) objectFolderPathForFile(file string) (string, error) { + rel, err := filepath.Rel(p.Config.Source, file) + + if err != nil { + return "", err + } + + return path.Join(p.objectFolderPath(), rel), nil +} + +// uploadFile uploads the file, relative to plugin source directory, to dst using global bucket. // To get a more robust upload use retryUpload instead. -func (p *Plugin) uploadFile(dst, file string) error { +func (p *Plugin) uploadFile(file string) error { r, gz, err := p.gzipper(file) if err != nil { @@ -156,13 +176,11 @@ func (p *Plugin) uploadFile(dst, file string) error { } defer r.Close() - rel, err := filepath.Rel(p.Config.Source, file) + name, err := p.objectFolderPathForFile(file) if err != nil { return err } - - name := path.Join(p.Config.Target, rel) w := p.bucket.Object(name).NewWriter(context.Background()) w.CacheControl = p.Config.CacheControl w.Metadata = p.Config.Metadata diff --git a/plugin_test.go b/plugin_test.go index f6747ff..650238f 100644 --- a/plugin_test.go +++ b/plugin_test.go @@ -130,7 +130,7 @@ func TestUploadFile(t *testing.T) { client, _ := storage.NewClient(context.Background(), option.WithHTTPClient(hc)) plugin.bucket = client.Bucket("bucket") - err := plugin.uploadFile("file", filepath.Join(wdir, "file")) + err = plugin.uploadFile(filepath.Join(wdir, "file")) switch { case test.ok && err != nil: @@ -154,18 +154,7 @@ func TestRun(t *testing.T) { writeFile(t, subdir, "file.css", []byte("sub style")) writeFile(t, subdir, "file.bin", []byte("rubbish")) - files := map[string]*struct { - ctype string - body []byte - gzip bool - }{ - "dir/file.txt": {"text/plain", []byte("text"), false}, - "dir/file.js": {"application/javascript", []byte("javascript"), true}, - "dir/sub/file.css": {"text/css", []byte("sub style"), false}, - } - plugin.Config.Source = wdir + "/upload" - plugin.Config.Target = "bucket/dir/" plugin.Config.Ignore = "sub/*.bin" plugin.Config.Gzip = []string{"js"} plugin.Config.CacheControl = "public,max-age=10" @@ -173,92 +162,153 @@ func TestRun(t *testing.T) { acls := []storage.ACLRule{storage.ACLRule{Entity: "allUsers", Role: "READER"}} plugin.Config.ACL = []string{fmt.Sprintf("%s:%s", acls[0].Entity, acls[0].Role)} - var seenMu sync.Mutex // guards seen - seen := make(map[string]struct{}, len(files)) + type filesMapEntry struct { + ctype string + body []byte + gzip bool + } - rt := &fakeTransport{func(r *http.Request) (resp *http.Response, e error) { - resp = &http.Response{ - Body: ioutil.NopCloser(strings.NewReader(`{"name": "fake"}`)), - Proto: "HTTP/1.0", - ProtoMajor: 1, - ProtoMinor: 0, - StatusCode: http.StatusOK, - } + testCases := []struct { + name string + files map[string]*filesMapEntry + pluginTarget string + pluginFolder string + }{ + { + name: "target with bucket only", + files: map[string]*filesMapEntry{ + "file.txt": {"text/plain", []byte("text"), false}, + "file.js": {"application/javascript", []byte("javascript"), true}, + "sub/file.css": {"text/css", []byte("sub style"), false}, + }, + pluginTarget: "bucket", + }, + { + name: "target with bucket and subdir", + files: map[string]*filesMapEntry{ + "dir/file.txt": {"text/plain", []byte("text"), false}, + "dir/file.js": {"application/javascript", []byte("javascript"), true}, + "dir/sub/file.css": {"text/css", []byte("sub style"), false}, + }, + pluginTarget: "bucket/dir", + }, + { + name: "target with bucket and folder", + files: map[string]*filesMapEntry{ + "folderpath/file.txt": {"text/plain", []byte("text"), false}, + "folderpath/file.js": {"application/javascript", []byte("javascript"), true}, + "folderpath/sub/file.css": {"text/css", []byte("sub style"), false}, + }, + pluginTarget: "bucket", + pluginFolder: "folderpath", + }, + { + name: "target with bucket, subdir and folder", + files: map[string]*filesMapEntry{ + "dir/folderpath/foldersub/file.txt": {"text/plain", []byte("text"), false}, + "dir/folderpath/foldersub/file.js": {"application/javascript", []byte("javascript"), true}, + "dir/folderpath/foldersub/sub/file.css": {"text/css", []byte("sub style"), false}, + }, + pluginTarget: "bucket/dir", + pluginFolder: "folderpath/foldersub", + }, + } - if !strings.HasSuffix(r.URL.Path, "/bucket/o") { - t.Errorf("r.URL.Path = %q; want /bucket/o suffix", r.URL.Path) - } - _, mp, err := mime.ParseMediaType(r.Header.Get("content-type")) - if err != nil { - t.Errorf("ParseMediaType: %v", err) - return - } - mr := multipart.NewReader(r.Body, mp["boundary"]) + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + var seenMu sync.Mutex // guards seen + seen := make(map[string]struct{}, len(testCase.files)) - // metadata - p, err := mr.NextPart() - if err != nil { - t.Errorf("meta NextPart: %v", err) - return - } - var attrs storage.ObjectAttrs - if err := json.NewDecoder(p).Decode(&attrs); err != nil { - t.Errorf("meta json: %v", err) - return - } - seenMu.Lock() - seen[attrs.Name] = struct{}{} - seenMu.Unlock() - obj := files[attrs.Name] - if obj == nil { - t.Errorf("unexpected obj: %+v", attrs) - return - } + rt := &fakeTransport{func(r *http.Request) (resp *http.Response, e error) { + resp = &http.Response{ + Body: ioutil.NopCloser(strings.NewReader(`{"name": "fake"}`)), + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + StatusCode: http.StatusOK, + } - if attrs.Bucket != "bucket" { - t.Errorf("attrs.Bucket = %q; want bucket", attrs.Bucket) - } - if attrs.CacheControl != plugin.Config.CacheControl { - t.Errorf("attrs.CacheControl = %q; want %q", attrs.CacheControl, plugin.Config.CacheControl) - } - if obj.gzip && attrs.ContentEncoding != "gzip" { - t.Errorf("attrs.ContentEncoding = %q; want gzip", attrs.ContentEncoding) - } - if !strings.HasPrefix(attrs.ContentType, obj.ctype) { - t.Errorf("attrs.ContentType = %q; want %q", attrs.ContentType, obj.ctype) - } - if !reflect.DeepEqual(attrs.ACL, acls) { - t.Errorf("attrs.ACL = %v; want %v", attrs.ACL, acls) - } - if !reflect.DeepEqual(attrs.Metadata, plugin.Config.Metadata) { - t.Errorf("attrs.Metadata = %+v; want %+v", attrs.Metadata, plugin.Config.Metadata) - } + if !strings.HasSuffix(r.URL.Path, "/bucket/o") { + t.Errorf("r.URL.Path = %q; want /bucket/o suffix", r.URL.Path) + } + _, mp, err := mime.ParseMediaType(r.Header.Get("content-type")) + if err != nil { + t.Errorf("ParseMediaType: %v", err) + return + } + mr := multipart.NewReader(r.Body, mp["boundary"]) - // media - p, err = mr.NextPart() - if err != nil { - t.Errorf("media NextPart: %v", err) - return - } - b, _ := ioutil.ReadAll(p) - if attrs.ContentEncoding == "gzip" { - b = gunzip(t, b) - } - if bytes.Compare(b, obj.body) != 0 { - t.Errorf("media b = %q; want %q", b, obj.body) - } - return - }} + // metadata + p, err := mr.NextPart() + if err != nil { + t.Errorf("meta NextPart: %v", err) + return + } + var attrs storage.ObjectAttrs + if err := json.NewDecoder(p).Decode(&attrs); err != nil { + t.Errorf("meta json: %v", err) + return + } + seenMu.Lock() + seen[attrs.Name] = struct{}{} + seenMu.Unlock() + obj := testCase.files[attrs.Name] + if obj == nil { + t.Errorf("unexpected obj: %+v", attrs) + return + } - hc := &http.Client{Transport: rt} - client, err := storage.NewClient(context.Background(), option.WithHTTPClient(hc)) - if err != nil { - t.Fatal(err) - } - plugin.Exec(client) - for k := range files { - if _, ok := seen[k]; !ok { - t.Errorf("%s didn't get uploaded", k) - } + if attrs.Bucket != "bucket" { + t.Errorf("attrs.Bucket = %q; want bucket", attrs.Bucket) + } + if attrs.CacheControl != plugin.Config.CacheControl { + t.Errorf("attrs.CacheControl = %q; want %q", attrs.CacheControl, plugin.Config.CacheControl) + } + if obj.gzip && attrs.ContentEncoding != "gzip" { + t.Errorf("attrs.ContentEncoding = %q; want gzip", attrs.ContentEncoding) + } + if !strings.HasPrefix(attrs.ContentType, obj.ctype) { + t.Errorf("attrs.ContentType = %q; want %q", attrs.ContentType, obj.ctype) + } + if !reflect.DeepEqual(attrs.ACL, acls) { + t.Errorf("attrs.ACL = %v; want %v", attrs.ACL, acls) + } + if !reflect.DeepEqual(attrs.Metadata, plugin.Config.Metadata) { + t.Errorf("attrs.Metadata = %+v; want %+v", attrs.Metadata, plugin.Config.Metadata) + } + + // media + p, err = mr.NextPart() + if err != nil { + t.Errorf("media NextPart: %v", err) + return + } + b, _ := ioutil.ReadAll(p) + if attrs.ContentEncoding == "gzip" { + b = gunzip(t, b) + } + if bytes.Compare(b, obj.body) != 0 { + t.Errorf("media b = %q; want %q", b, obj.body) + } + return + }} + + hc := &http.Client{Transport: rt} + client, err := storage.NewClient(context.Background(), option.WithHTTPClient(hc)) + if err != nil { + t.Fatal(err) + } + + plugin.Config.Target = testCase.pluginTarget + plugin.Config.Folder = testCase.pluginFolder + + plugin.Exec(client) + + for k := range testCase.files { + if _, ok := seen[k]; !ok { + t.Errorf("%s didn't get uploaded", k) + } + } + }) } }