Skip to content

Commit

Permalink
feat(golang): add license parsing from vendor dirs
Browse files Browse the repository at this point in the history
  • Loading branch information
dschmidt committed Dec 12, 2024
1 parent 02f9350 commit 098d82d
Show file tree
Hide file tree
Showing 6 changed files with 153 additions and 22 deletions.
2 changes: 2 additions & 0 deletions cmd/syft/internal/options/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
Golang: golang.DefaultCatalogerConfig().
WithSearchLocalModCacheLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.SearchLocalModCacheLicenses)).
WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
WithSearchLocalVendorLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.SearchLocalVendorLicenses)).
WithLocalVendorDir(cfg.Golang.LocalVendorDir).
WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.SearchRemoteLicenses)).
WithProxy(cfg.Golang.Proxy).
WithNoProxy(cfg.Golang.NoProxy).
Expand Down
7 changes: 7 additions & 0 deletions cmd/syft/internal/options/golang.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
type golangConfig struct {
SearchLocalModCacheLicenses *bool `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
LocalModCacheDir string `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
SearchLocalVendorLicenses *bool `json:"search-local-vendor-licenses" yaml:"search-local-vendor-licenses" mapstructure:"search-local-vendor-licenses"`
LocalVendorDir string `json:"local-vendor-dir" yaml:"local-vendor-dir" mapstructure:"local-vendor-dir"`
SearchRemoteLicenses *bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
Proxy string `json:"proxy" yaml:"proxy" mapstructure:"proxy"`
NoProxy string `json:"no-proxy" yaml:"no-proxy" mapstructure:"no-proxy"`
Expand All @@ -24,6 +26,9 @@ func (o *golangConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
descriptions.Add(&o.SearchLocalModCacheLicenses, `search for go package licences in the GOPATH of the system running Syft, note that this is outside the
container filesystem and potentially outside the root of a local directory scan`)
descriptions.Add(&o.LocalModCacheDir, `specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod`)
descriptions.Add(&o.SearchLocalVendorLicenses, `search for go package licences in the vendor folder on the system running Syft, note that this is outside the
container filesystem and potentially outside the root of a local directory scan`)
descriptions.Add(&o.LocalVendorDir, `specify an explicit go vendor directory, if unset this defaults to ./vendor`)
descriptions.Add(&o.SearchRemoteLicenses, `search for go package licences by retrieving the package from a network proxy`)
descriptions.Add(&o.Proxy, `remote proxy to use when retrieving go packages from the network,
if unset this defaults to $GOPROXY followed by https://proxy.golang.org`)
Expand All @@ -49,6 +54,8 @@ func defaultGolangConfig() golangConfig {
return golangConfig{
SearchLocalModCacheLicenses: nil, // this defaults to false, which is the API default
LocalModCacheDir: def.LocalModCacheDir,
SearchLocalVendorLicenses: nil, // this defaults to false, which is the API default
LocalVendorDir: def.LocalVendorDir,
SearchRemoteLicenses: nil, // this defaults to false, which is the API default
Proxy: strings.Join(def.Proxies, ","),
NoProxy: strings.Join(def.NoProxy, ","),
Expand Down
25 changes: 25 additions & 0 deletions syft/pkg/cataloger/golang/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ var (
type CatalogerConfig struct {
SearchLocalModCacheLicenses bool `yaml:"search-local-mod-cache-licenses" json:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
LocalModCacheDir string `yaml:"local-mod-cache-dir" json:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
SearchLocalVendorLicenses bool `yaml:"search-local-vendor-licenses" json:"search-local-vendor-licenses" mapstructure:"search-local-vendor-licenses"`
LocalVendorDir string `yaml:"local-vendor-dir" json:"local-vendor-dir" mapstructure:"local-vendor-dir"`
SearchRemoteLicenses bool `yaml:"search-remote-licenses" json:"search-remote-licenses" mapstructure:"search-remote-licenses"`
Proxies []string `yaml:"proxies,omitempty" json:"proxies,omitempty" mapstructure:"proxies"`
NoProxy []string `yaml:"no-proxy,omitempty" json:"no-proxy,omitempty" mapstructure:"no-proxy"`
Expand All @@ -42,6 +44,7 @@ func DefaultCatalogerConfig() CatalogerConfig {
g := CatalogerConfig{
MainModuleVersion: DefaultMainModuleVersionConfig(),
LocalModCacheDir: defaultGoModDir(),
LocalVendorDir: defaultGoVendorDir(),
}

// first process the proxy settings
Expand Down Expand Up @@ -71,6 +74,15 @@ func DefaultCatalogerConfig() CatalogerConfig {
return g
}

func defaultGoVendorDir() string {
cwd, err := os.Getwd()
if err != nil {
return ""
}

return filepath.Join(cwd, "vendor")
}

// defaultGoModDir returns $GOPATH/pkg/mod or $HOME/go/pkg/mod based on environment variables available
func defaultGoModDir() string {
goPath := os.Getenv("GOPATH")
Expand Down Expand Up @@ -108,6 +120,19 @@ func (g CatalogerConfig) WithLocalModCacheDir(input string) CatalogerConfig {
return g
}

func (g CatalogerConfig) WithSearchLocalVendorLicenses(input bool) CatalogerConfig {
g.SearchLocalVendorLicenses = input
return g
}

func (g CatalogerConfig) WithLocalVendorDir(input string) CatalogerConfig {
if input == "" {
return g
}
g.LocalVendorDir = input
return g
}

func (g CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig {
g.SearchRemoteLicenses = input
return g
Expand Down
33 changes: 22 additions & 11 deletions syft/pkg/cataloger/golang/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ import (

func Test_Config(t *testing.T) {
type opts struct {
local bool
cacheDir string
remote bool
proxy string
noProxy string
local bool
cacheDir string
vendorDir string
remote bool
proxy string
noProxy string
}

homedirCacheDisabled := homedir.DisableCache
Expand Down Expand Up @@ -45,10 +46,15 @@ func Test_Config(t *testing.T) {
"GOPRIVATE": "my.private",
"GONOPROXY": "no.proxy",
},
opts: opts{},
opts: opts{
// defaults to $cwd/vendor, we need to set it to make the output predictable
vendorDir: "/vendor",
},
expected: CatalogerConfig{
SearchLocalModCacheLicenses: false,
LocalModCacheDir: filepath.Join("/go", "pkg", "mod"),
SearchLocalVendorLicenses: false,
LocalVendorDir: "/vendor",
SearchRemoteLicenses: false,
Proxies: []string{"https://my.proxy"},
NoProxy: []string{"my.private", "no.proxy"},
Expand All @@ -64,15 +70,18 @@ func Test_Config(t *testing.T) {
"GONOPROXY": "no.proxy",
},
opts: opts{
local: true,
cacheDir: "/go-cache",
remote: true,
proxy: "https://alt.proxy,direct",
noProxy: "alt.no.proxy",
local: true,
cacheDir: "/go-cache",
vendorDir: "/vendor",
remote: true,
proxy: "https://alt.proxy,direct",
noProxy: "alt.no.proxy",
},
expected: CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: "/go-cache",
SearchLocalVendorLicenses: true,
LocalVendorDir: "/vendor",
SearchRemoteLicenses: true,
Proxies: []string{"https://alt.proxy", "direct"},
NoProxy: []string{"alt.no.proxy"},
Expand All @@ -92,6 +101,8 @@ func Test_Config(t *testing.T) {
got := DefaultCatalogerConfig().
WithSearchLocalModCacheLicenses(test.opts.local).
WithLocalModCacheDir(test.opts.cacheDir).
WithSearchLocalVendorLicenses(test.opts.local).
WithLocalVendorDir(test.opts.vendorDir).
WithSearchRemoteLicenses(test.opts.remote).
WithProxy(test.opts.proxy).
WithNoProxy(test.opts.noProxy)
Expand Down
33 changes: 25 additions & 8 deletions syft/pkg/cataloger/golang/licenses.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type goLicenseResolver struct {
catalogerName string
opts CatalogerConfig
localModCacheDir fs.FS
localVendorDir fs.FS
licenseCache cache.Resolver[[]goLicense]
lowerLicenseFileNames *strset.Set
}
Expand All @@ -52,10 +53,16 @@ func newGoLicenseResolver(catalogerName string, opts CatalogerConfig) goLicenseR
localModCacheDir = os.DirFS(opts.LocalModCacheDir)
}

var localVendorDir fs.FS
if opts.SearchLocalVendorLicenses && opts.LocalVendorDir != "" {
localVendorDir = os.DirFS(opts.LocalVendorDir)
}

return goLicenseResolver{
catalogerName: catalogerName,
opts: opts,
localModCacheDir: localModCacheDir,
localVendorDir: localVendorDir,
licenseCache: cache.GetResolverCachingErrors[[]goLicense]("golang", "v1"),
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
}
Expand Down Expand Up @@ -91,7 +98,15 @@ func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Sc

// look in the local host mod directory...
if c.opts.SearchLocalModCacheLicenses {
goLicenses, err = c.getLicensesFromLocal(ctx, scanner, moduleName, moduleVersion)
goLicenses, err = c.getLicensesFromLocal(ctx, scanner, c.localModCacheDir, moduleDirCache(moduleName, moduleVersion))
if err != nil || len(goLicenses) > 0 {
return toPkgLicenses(goLicenses), err
}
}

// look in the local vendor directory...
if c.opts.SearchLocalVendorLicenses {
goLicenses, err = c.getLicensesFromLocal(ctx, scanner, c.localVendorDir, moduleDirVendor(moduleName))
if err != nil || len(goLicenses) > 0 {
return toPkgLicenses(goLicenses), err
}
Expand All @@ -105,23 +120,21 @@ func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Sc
return toPkgLicenses(goLicenses), err
}

func (c *goLicenseResolver) getLicensesFromLocal(ctx context.Context, scanner licenses.Scanner, moduleName, moduleVersion string) ([]goLicense, error) {
if c.localModCacheDir == nil {
func (c *goLicenseResolver) getLicensesFromLocal(ctx context.Context, scanner licenses.Scanner, moduleDir fs.FS, moduleSubdir string) ([]goLicense, error) {
if moduleDir == nil {
return nil, nil
}

subdir := moduleDir(moduleName, moduleVersion)

// get the local subdirectory containing the specific go module
dir, err := fs.Sub(c.localModCacheDir, subdir)
dir, err := fs.Sub(moduleDir, moduleSubdir)
if err != nil {
return nil, err
}

// if we're running against a directory on the filesystem, it may not include the
// user's homedir / GOPATH, so we defer to using the localModCacheResolver
// we use $GOPATH/pkg/mod to avoid leaking information about the user's system
return c.findLicensesInFS(ctx, scanner, "file://$GOPATH/pkg/mod/"+subdir+"/", dir)
return c.findLicensesInFS(ctx, scanner, "file://$GOPATH/pkg/mod/"+moduleSubdir+"/", dir)
}

func (c *goLicenseResolver) getLicensesFromRemote(ctx context.Context, scanner licenses.Scanner, moduleName, moduleVersion string) ([]goLicense, error) {
Expand Down Expand Up @@ -221,10 +234,14 @@ func (c *goLicenseResolver) parseLicenseFromLocation(ctx context.Context, scanne
return out, nil
}

func moduleDir(moduleName, moduleVersion string) string {
func moduleDirCache(moduleName, moduleVersion string) string {
return fmt.Sprintf("%s@%s", processCaps(moduleName), moduleVersion)
}

func moduleDirVendor(moduleName string) string {
return processCaps(moduleName)
}

func requireCollection[T any](licenses []T) []T {
if licenses == nil {
return make([]T, 0)
Expand Down
75 changes: 72 additions & 3 deletions syft/pkg/cataloger/golang/licenses_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"github.com/anchore/syft/syft/pkg"
)

func Test_LocalLicenseSearch(t *testing.T) {
func Test_LocalModCacheLicenseSearch(t *testing.T) {
loc1 := file.NewLocation("github.com/someorg/[email protected]/LICENSE")
loc2 := file.NewLocation("github.com/!cap!o!r!g/[email protected]/LICENSE.txt")
loc3 := file.NewLocation("github.com/someorg/[email protected]/LiCeNsE.tXt")
Expand Down Expand Up @@ -92,6 +92,75 @@ func Test_LocalLicenseSearch(t *testing.T) {
}
}

func Test_LocalVendorLicenseSearch(t *testing.T) {
loc1 := file.NewLocation("github.com/someorg/somename/LICENSE")
loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project/LICENSE.txt")
loc3 := file.NewLocation("github.com/someorg/strangelicense/LiCeNsE.tXt")

licenseScanner := licenses.TestingOnlyScanner()

tests := []struct {
name string
version string
expected pkg.License
}{
{
name: "github.com/someorg/somename",
version: "v0.3.2",
expected: pkg.License{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc1.RealPath},
Locations: file.NewLocationSet(),
},
},
{
name: "github.com/CapORG/CapProject",
version: "v4.111.5",
expected: pkg.License{
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc2.RealPath},
Locations: file.NewLocationSet(),
},
},
{
name: "github.com/someorg/strangelicense",
version: "v1.2.3",
expected: pkg.License{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: license.Concluded,
URLs: []string{"file://$GOPATH/pkg/mod/" + loc3.RealPath},
Locations: file.NewLocationSet(),
},
},
}

wd, err := os.Getwd()
require.NoError(t, err)

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
l := newGoLicenseResolver(
"",
CatalogerConfig{
SearchLocalVendorLicenses: true,
LocalVendorDir: filepath.Join(wd, "test-fixtures", "licenses-vendor"),
},
)
lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version)
require.NoError(t, err)

require.Len(t, lics, 1)

require.Equal(t, test.expected, lics[0])
})
}
}

func Test_RemoteProxyLicenseSearch(t *testing.T) {
loc1 := file.NewLocation("github.com/someorg/[email protected]/LICENSE")
loc2 := file.NewLocation("github.com/!cap!o!r!g/[email protected]/LICENSE.txt")
Expand All @@ -117,7 +186,7 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
for _, f := range entries {
// the zip files downloaded contain a path to the repo that somewhat matches where it ends up on disk,
// so prefix entries with something similar
writer, err := archive.Create(path.Join(moduleDir(modPath, modVersion), f.Name()))
writer, err := archive.Create(path.Join(moduleDirCache(modPath, modVersion), f.Name()))
require.NoError(t, err)
contents, err := os.ReadFile(filepath.Join(testDir, f.Name()))
require.NoError(t, err)
Expand Down Expand Up @@ -307,7 +376,7 @@ func Test_noLocalGoModDir(t *testing.T) {
SearchLocalModCacheLicenses: true,
LocalModCacheDir: test.dir,
})
_, err := resolver.getLicensesFromLocal(context.Background(), licenseScanner, "mod", "ver")
_, err := resolver.getLicensesFromLocal(context.Background(), licenseScanner, resolver.localModCacheDir, moduleDirCache("mod", "ver"))
test.wantErr(t, err)
})
}
Expand Down

0 comments on commit 098d82d

Please sign in to comment.