forked from meltwater/drone-cache
-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: [CI-11277]: Support Signed URLs for Cache Intelligence workflows (
#93) * fix: [CI-11277]: Support Signed URLs for Cache Intelligence workflows * fix * fix
- Loading branch information
1 parent
0a9466e
commit c6fcb41
Showing
9 changed files
with
265 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package harness | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| ) | ||
|
|
||
| // Error is a custom error struct | ||
| type Error struct { | ||
| Code int | ||
| Message string | ||
| } | ||
|
|
||
| func (e *Error) Error() string { | ||
| return fmt.Sprintf("%d: %s", e.Code, e.Message) | ||
| } | ||
|
|
||
| // Client defines a cache service client. | ||
| type Client interface { | ||
| GetUploadURL(ctx context.Context, key string) (string, error) | ||
|
|
||
| GetDownloadURL(ctx context.Context, key string) (string, error) | ||
|
|
||
| GetExistsURL(ctx context.Context, key string) (string, error) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package harness | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "strings" | ||
| ) | ||
|
|
||
| var _ Client = (*HTTPClient)(nil) | ||
|
|
||
| const ( | ||
| RestoreEndpoint = "/cache/intel/download?accountId=%s&cacheKey=%s" | ||
| StoreEndpoint = "/cache/intel/upload?accountId=%s&cacheKey=%s" | ||
| ExistsEndpoint = "/cache/intel/exists?accountId=%s&cacheKey=%s" | ||
| ) | ||
|
|
||
| // NewHTTPClient returns a new HTTPClient. | ||
| func New(endpoint, accountID, bearerToken string, skipverify bool) *HTTPClient { | ||
| endpoint = strings.TrimSuffix(endpoint, "/") | ||
| client := &HTTPClient{ | ||
| Endpoint: endpoint, | ||
| BearerToken: bearerToken, | ||
| AccountID: accountID, | ||
| Client: &http.Client{ | ||
| CheckRedirect: func(*http.Request, []*http.Request) error { | ||
| return http.ErrUseLastResponse | ||
| }, | ||
| }, | ||
| } | ||
| return client | ||
| } | ||
|
|
||
| // HTTPClient provides an http service client. | ||
| type HTTPClient struct { | ||
| Client *http.Client | ||
| Endpoint string | ||
| AccountID string | ||
| BearerToken string | ||
| } | ||
|
|
||
| // getUploadURL will get the 'put' presigned url from cache service | ||
| func (c *HTTPClient) GetUploadURL(ctx context.Context, key string) (string, error) { | ||
| path := fmt.Sprintf(StoreEndpoint, c.AccountID, key) | ||
| return c.getLink(ctx, c.Endpoint+path) | ||
| } | ||
|
|
||
| // getDownloadURL will get the 'get' presigned url from cache service | ||
| func (c *HTTPClient) GetDownloadURL(ctx context.Context, key string) (string, error) { | ||
| path := fmt.Sprintf(RestoreEndpoint, c.AccountID, key) | ||
| return c.getLink(ctx, c.Endpoint+path) | ||
| } | ||
|
|
||
| // getExistsURL will get the 'exists' presigned url from cache service | ||
| func (c *HTTPClient) GetExistsURL(ctx context.Context, key string) (string, error) { | ||
| path := fmt.Sprintf(ExistsEndpoint, c.AccountID, key) | ||
| return c.getLink(ctx, c.Endpoint+path) | ||
| } | ||
|
|
||
| func (c *HTTPClient) getLink(ctx context.Context, path string) (string, error) { | ||
| req, err := http.NewRequestWithContext(ctx, "GET", path, nil) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| if c.BearerToken != "" { | ||
| req.Header.Add("X-Harness-Token", c.BearerToken) | ||
| } | ||
|
|
||
| resp, err := c.client().Do(req) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| if resp.StatusCode != http.StatusOK { | ||
| return "", fmt.Errorf("failed to get link with status %d", resp.StatusCode) | ||
| } | ||
| body, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return string(body), nil | ||
| } | ||
|
|
||
| func (c *HTTPClient) client() *http.Client { | ||
| return c.Client | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| package harness | ||
|
|
||
| // Config is a structure to store harness backend configuration. | ||
| type Config struct { | ||
| AccountID string | ||
| Token string | ||
| ServerBaseURL string | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package harness | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
|
|
||
| "github.com/go-kit/kit/log" | ||
| "github.com/meltwater/drone-cache/harness" | ||
| "github.com/meltwater/drone-cache/internal" | ||
| "github.com/meltwater/drone-cache/storage/common" | ||
| ) | ||
|
|
||
| type Backend struct { | ||
| logger log.Logger | ||
| token string | ||
| client harness.Client | ||
| } | ||
|
|
||
| // New creates an Harness backend. | ||
| func New(l log.Logger, c Config, debug bool) (*Backend, error) { | ||
| cacheClient := harness.New(c.ServerBaseURL, c.AccountID, c.Token, false) | ||
| backend := &Backend{ | ||
| logger: l, | ||
| token: c.Token, | ||
| client: cacheClient, | ||
| } | ||
| return backend, nil | ||
| } | ||
|
|
||
| func (b *Backend) Get(ctx context.Context, key string, w io.Writer) error { | ||
| preSignedURL, err := b.client.GetDownloadURL(ctx, key) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| res, err := b.do(ctx, "GET", preSignedURL, nil) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer internal.CloseWithErrLogf(b.logger, res.Body, "response body, close defer") | ||
| if res.StatusCode != http.StatusOK { | ||
| return fmt.Errorf("received status code %d from presigned get url", res.StatusCode) | ||
| } | ||
| _, err = io.Copy(w, res.Body) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (b *Backend) Put(ctx context.Context, key string, r io.Reader) error { | ||
| preSignedURL, err := b.client.GetUploadURL(ctx, key) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| res, err := b.do(ctx, "PUT", preSignedURL, r) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer internal.CloseWithErrLogf(b.logger, res.Body, "response body, close defer") | ||
| if res.StatusCode != http.StatusOK { | ||
| return fmt.Errorf("received status code %d from presigned put url", res.StatusCode) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (b *Backend) Exists(ctx context.Context, key string) (bool, error) { | ||
| preSignedURL, err := b.client.GetExistsURL(ctx, key) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| res, err := b.do(ctx, "HEAD", preSignedURL, nil) | ||
| if err != nil { | ||
| return false, nil | ||
| } | ||
| defer internal.CloseWithErrLogf(b.logger, res.Body, "response body, close defer") | ||
| if res.StatusCode == http.StatusNotFound { | ||
| return false, nil | ||
| } else if res.StatusCode != http.StatusOK { | ||
| return false, fmt.Errorf("unexpected status code %d", res.StatusCode) | ||
| } | ||
|
|
||
| return res.Header.Get("ETag") != "", nil | ||
| } | ||
|
|
||
| func (b *Backend) List(ctx context.Context, key string) ([]common.FileEntry, error) { | ||
| // not implemented | ||
| return nil, errors.New("list operation not implemented") | ||
| } | ||
|
|
||
| func (b *Backend) do(ctx context.Context, method, url string, body io.Reader) (*http.Response, error) { | ||
| var ( | ||
| buffer []byte | ||
| err error | ||
| ) | ||
| if body != nil { | ||
| buffer, err = io.ReadAll(body) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
| req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(buffer)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| httpClient := http.Client{} | ||
| res, err := httpClient.Do(req) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return res, nil | ||
| } |