Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Let some people know they can try out newer versions, if we know they aren't bad! #225

Draft
wants to merge 12 commits into
base: main
Choose a base branch
from
Draft

Let some people know they can try out newer versions, if we know they aren't bad! #225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
6a5ca1d
You bet I busted some stuff, but lets start seeing what people think
DarthHater Apr 13, 2021
26acba3
Add a comment on what it fixes
DarthHater Apr 13, 2021
3c452f5
add in some color, why not
DarthHater Apr 13, 2021
2d6749e
Simplify this a bit
DarthHater Apr 13, 2021
a514a96
Fix up CSV
DarthHater Apr 13, 2021
bd7f252
Add some comments to more rapidly explain what I did
DarthHater Apr 13, 2021
9feac53
Various
DarthHater Apr 13, 2021
b07433a
Update packages/packages.go
DarthHater Apr 13, 2021
453613b
Rename Projects to Dependency, and do some refactoring while I am at it
DarthHater Apr 14, 2021
45979ee
Merge branch 'NancyHelpsYou' of github.com:sonatype-nexus-community/n…
DarthHater Apr 14, 2021
af2d3f2
Turns out this DOES need to be public, at least for now
DarthHater Apr 14, 2021
e9e94b0
Fix tests, more to go
DarthHater Apr 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ require (
github.com/sdboyer/constext v0.0.0-20170321163424-836a14457353 // indirect
github.com/shopspring/decimal v1.2.0
github.com/sirupsen/logrus v1.6.0
github.com/sonatype-nexus-community/go-sona-types v0.0.12
github.com/sonatype-nexus-community/go-sona-types v0.1.1
github.com/spf13/afero v1.3.4 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/cobra v1.0.0
Expand All @@ -37,10 +37,10 @@ require (
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
)

// fix vulnerability: CVE-2020-15114 in etcd v3.3.13+incompatible
replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible
// // fix vulnerability: CVE-2020-15114 in etcd v3.3.13+incompatible
// replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible
Comment on lines +40 to +41
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have these commented out for easy local testing of vulnerable dependencies


// fix vulnerability: CVE-2021-3121 in github.com/gogo/protobuf v1.2.1
replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
// // fix vulnerability: CVE-2021-3121 in github.com/gogo/protobuf v1.2.1
// replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
Comment on lines +43 to +44
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have these commented out for easy local testing of vulnerable dependencies


go 1.13
139 changes: 139 additions & 0 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

46 changes: 16 additions & 30 deletions internal/audit/auditlog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,51 +20,37 @@ import (
"os"

log "github.com/sirupsen/logrus"
"github.com/sonatype-nexus-community/go-sona-types/ossindex/types"
ossIndexTypes "github.com/sonatype-nexus-community/go-sona-types/ossindex/types"
"github.com/sonatype-nexus-community/nancy/buildversion"
"github.com/sonatype-nexus-community/nancy/types"
)

// LogResults will given a number of expected results and the results themselves, log the
// results.
func LogResults(formatter log.Formatter, packageCount int, coordinates []types.Coordinate, invalidCoordinates []types.Coordinate, exclusions []string) int {
vulnerableCount := 0
func LogResults(formatter log.Formatter, packageCount int, coordinates map[string]ossIndexTypes.Coordinate, invalidCoordinates []ossIndexTypes.Coordinate, vulnerableCoordinates map[string]types.Dependency, exclusions []string) int {

for _, c := range coordinates {
c.ExcludeVulnerabilities(exclusions)
if exclusions == nil {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I moved this further up because it seemed odd it would make this AFTER we try using exclusions?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so the only reason it was down there was b/c if you log it it would not work correctly i believe. So if you passed in nil we wanted it to present to the user as []. Its used in the ExcludeVulnerabilites but range over nil is noop ¯_(ツ)_/¯

https://github.com/sonatype-nexus-community/go-sona-types/blob/b469b94b5ebb013125d69c3524ba5d6f759eede0/ossindex/types/types.go#L103-L107

So really we only do it for presentation. If we can find a better way im cool with it. But moving it up or down doesn't mean much just has to be before the log statement at line 46 hits.

exclusions = make([]string, 0)
}

var auditedCoordinates []types.Coordinate
var vulnerableCoordinates []types.Coordinate

for i := 0; i < len(coordinates); i++ {
coordinate := coordinates[i]
if coordinate.IsVulnerable() {
vulnerableCount++
vulnerableCoordinates = append(vulnerableCoordinates, coordinate)
}
auditedCoordinates = append(auditedCoordinates, coordinate)
for _, c := range vulnerableCoordinates {
c.Coordinate.ExcludeVulnerabilities(exclusions)
}

if invalidCoordinates == nil {
invalidCoordinates = make([]types.Coordinate, 0)
}
if exclusions == nil {
exclusions = make([]string, 0)
}
if vulnerableCoordinates == nil {
vulnerableCoordinates = make([]types.Coordinate, 0)
invalidCoordinates = make([]ossIndexTypes.Coordinate, 0)
}

log.SetFormatter(formatter)
log.SetOutput(os.Stdout)
log.WithFields(log.Fields{
"exclusions": exclusions,
"num_audited": packageCount,
"num_vulnerable": vulnerableCount,
"audited": auditedCoordinates,
"vulnerable": vulnerableCoordinates,
"invalid": invalidCoordinates,
"version": buildversion.BuildVersion,
"exclusions": exclusions,
"num_audited": packageCount,
"audited": coordinates,
"vulnerable": vulnerableCoordinates,
"invalid": invalidCoordinates,
"version": buildversion.BuildVersion,
}).Info("")

return vulnerableCount
return len(vulnerableCoordinates)
}
79 changes: 44 additions & 35 deletions internal/audit/auditlogtextformatter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ import (
"github.com/logrusorgru/aurora"
"github.com/shopspring/decimal"
. "github.com/sirupsen/logrus"
"github.com/sonatype-nexus-community/go-sona-types/ossindex/types"
ossIndexTypes "github.com/sonatype-nexus-community/go-sona-types/ossindex/types"
"github.com/sonatype-nexus-community/nancy/types"
)

var (
Expand All @@ -47,7 +48,7 @@ type AuditLogTextFormatter struct {
NoColor bool
}

func logPackage(sb *strings.Builder, noColor bool, coordinate types.Coordinate) {
func logPackage(sb *strings.Builder, noColor bool, coordinate ossIndexTypes.Coordinate) {
au := aurora.NewAurora(!noColor)

sb.WriteString(
Expand All @@ -57,7 +58,7 @@ func logPackage(sb *strings.Builder, noColor bool, coordinate types.Coordinate)
)
}

func logInvalidSemVerWarning(sb *strings.Builder, noColor bool, quiet bool, invalidPurls []types.Coordinate) {
func logInvalidSemVerWarning(sb *strings.Builder, noColor bool, quiet bool, invalidPurls []ossIndexTypes.Coordinate) {
if !quiet {
if len(invalidPurls) > 0 {
au := aurora.NewAurora(!noColor)
Expand All @@ -76,19 +77,19 @@ func logInvalidSemVerWarning(sb *strings.Builder, noColor bool, quiet bool, inva
}
}

func logVulnerablePackage(sb *strings.Builder, noColor bool, coordinate types.Coordinate) {
func logVulnerablePackage(sb *strings.Builder, noColor bool, coordinate types.Dependency) {
au := aurora.NewAurora(!noColor)
sb.WriteString(fmt.Sprintf(
"%s\n%s \n",
au.Bold(au.Red(coordinate.Coordinates)).String(),
au.Red(strconv.Itoa(len(coordinate.Vulnerabilities))+" known vulnerabilities affecting installed version").String(),
au.Bold(au.Red(coordinate.Coordinate.Coordinates)).String(),
au.Red(strconv.Itoa(len(coordinate.Coordinate.Vulnerabilities))+" known vulnerabilities affecting installed version").String(),
))

sort.Slice(coordinate.Vulnerabilities, func(i, j int) bool {
return coordinate.Vulnerabilities[i].CvssScore.GreaterThan(coordinate.Vulnerabilities[j].CvssScore)
sort.Slice(coordinate.Coordinate.Vulnerabilities, func(i, j int) bool {
return coordinate.Coordinate.Vulnerabilities[i].CvssScore.GreaterThan(coordinate.Coordinate.Vulnerabilities[j].CvssScore)
})

for _, v := range coordinate.Vulnerabilities {
for _, v := range coordinate.Coordinate.Vulnerabilities {
if !v.Excluded {
t := table.NewWriter()
t.SetStyle(table.StyleBold)
Expand All @@ -102,6 +103,7 @@ func logVulnerablePackage(sb *strings.Builder, noColor bool, coordinate types.Co
t.AppendRow([]interface{}{"CVSS Vector", v.CvssVector})
t.AppendSeparator()
t.AppendRow([]interface{}{"Link for more info", v.Reference})

sb.WriteString(t.Render() + "\n")
}
}
Expand Down Expand Up @@ -134,66 +136,73 @@ func scoreAssessment(score decimal.Decimal) string {
return "Low"
}

func groupAndPrint(vulnerable []types.Coordinate, nonVulnerable []types.Coordinate, quiet bool, noColor bool, sb *strings.Builder) {
func groupAndPrint(vulnerableDependencies map[string]types.Dependency, nonVulnerable map[string]ossIndexTypes.Coordinate, quiet bool, noColor bool, sb *strings.Builder) {
if !quiet {
sb.WriteString("\n")
for _, v := range nonVulnerable {
logPackage(sb, noColor, v)
}
sb.WriteString(fmt.Sprintf("\n%d Non Vulnerable Packages\n\n", len(nonVulnerable)))
}
if len(vulnerable) > 0 {
for _, v := range vulnerable {
if len(vulnerableDependencies) > 0 {
for _, v := range vulnerableDependencies {
logVulnerablePackage(sb, noColor, v)
}
sb.WriteString(fmt.Sprintf("\n%d Vulnerable Packages\n\n", len(vulnerable)))
sb.WriteString(fmt.Sprintf("\n%d Vulnerable Packages\n\n", len(vulnerableDependencies)))
}
}

func (f AuditLogTextFormatter) Format(entry *Entry) ([]byte, error) {
auditedEntries := entry.Data["audited"]
invalidEntries := entry.Data["invalid"]
packageCount := entry.Data["num_audited"]
numVulnerable := entry.Data["num_vulnerable"]
vulnerableEntries := entry.Data["vulnerable"]
buildVersion := entry.Data["version"]
if auditedEntries != nil && invalidEntries != nil && packageCount != nil && numVulnerable != nil && buildVersion != nil {
auditedEntries := entry.Data["audited"].([]types.Coordinate)
invalidEntries := entry.Data["invalid"].([]types.Coordinate)
packageCount := entry.Data["num_audited"].(int)
numVulnerable := entry.Data["num_vulnerable"].(int)
if auditedEntries != nil && invalidEntries != nil && vulnerableEntries != nil && buildVersion != nil {
auditedEntries := entry.Data["audited"].(map[string]ossIndexTypes.Coordinate)
vulnerableEntries := entry.Data["vulnerable"].(map[string]types.Dependency)
invalidEntries := entry.Data["invalid"].([]ossIndexTypes.Coordinate)

var sb strings.Builder

w := tabwriter.NewWriter(&sb, 9, 3, 0, '\t', 0)
_ = w.Flush()

logInvalidSemVerWarning(&sb, f.NoColor, f.Quiet, invalidEntries)
nonVulnerablePackages, vulnerablePackages := splitPackages(auditedEntries)

groupAndPrint(vulnerablePackages, nonVulnerablePackages, f.Quiet, f.NoColor, &sb)
groupAndPrint(vulnerableEntries, auditedEntries, f.Quiet, f.NoColor, &sb)

au := aurora.NewAurora(!f.NoColor)

var updates []string
for _, v := range vulnerableEntries {
if v.UpdateCoordinate.Coordinates != "" && v.Update != nil {
var issueTitles []string
for _, v := range v.Coordinate.Vulnerabilities {
issueTitles = append(issueTitles, v.Cve)
}
comment := au.Green(fmt.Sprintf("// fix issues: %s in %s %s\n", strings.Join(issueTitles, ", "), v.Name, v.Version)).String()
replace := au.Blue(fmt.Sprintf("replace %s => %s %s\n\n", v.Update.Path, v.Update.Path, v.Update.Version)).String()
updates = append(updates, comment, replace)
}
}

if len(updates) > 0 {
sb.WriteString(au.Bold("I found some updated versions you can try out, that have no known vulnerabilities!\n\nTry the following in your go.mod file:\n\n").String())
for _, v := range updates {
sb.WriteString(au.Italic(v).String())
}
}

t := table.NewWriter()
t.SetStyle(table.StyleBold)
t.SetTitle("Summary")
t.AppendRow([]interface{}{"Audited Dependencies", strconv.Itoa(packageCount)})
t.AppendRow([]interface{}{"Audited Dependencies", strconv.Itoa(len(auditedEntries))})
t.AppendSeparator()
t.AppendRow([]interface{}{"Vulnerable Dependencies", au.Bold(au.Red(strconv.Itoa(numVulnerable)))})
t.AppendRow([]interface{}{"Vulnerable Dependencies", au.Bold(au.Red(strconv.Itoa(len(vulnerableEntries))))})
sb.WriteString(t.Render())
sb.WriteString("\n")

return []byte(sb.String()), nil
}
return nil, errors.New("fields passed did not match the expected values for an audit log. You should probably look at setting the formatter to something else")
}

func splitPackages(entries []types.Coordinate) (nonVulnerable []types.Coordinate, vulnerable []types.Coordinate) {
for _, v := range entries {
if v.IsVulnerable() {
vulnerable = append(vulnerable, v)
} else {
nonVulnerable = append(nonVulnerable, v)
}
}
return
}
Comment on lines -190 to -199
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This wasn't even being used, or rather it was, but we already had two lists of data. I removed it because LOL

13 changes: 6 additions & 7 deletions internal/audit/csvformatter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,20 @@ func (f CsvFormatter) Format(entry *Entry) ([]byte, error) {
// source of the official loggers.
auditedEntries := entry.Data["audited"]
invalidEntries := entry.Data["invalid"]
packageCount := entry.Data["num_audited"]
numVulnerable := entry.Data["num_vulnerable"]
buildVersion := entry.Data["version"]
vulnerableEntries := entry.Data["vulnerable"]

if auditedEntries != nil && invalidEntries != nil && packageCount != nil && numVulnerable != nil && buildVersion != nil {
if auditedEntries != nil && invalidEntries != nil && vulnerableEntries != nil && buildVersion != nil {
auditedEntries := entry.Data["audited"].([]types.Coordinate)
invalidEntries := entry.Data["invalid"].([]types.Coordinate)
packageCount := entry.Data["num_audited"].(int)
numVulnerable := entry.Data["num_vulnerable"].(int)
buildVersion := entry.Data["version"].(string)
numVulnerable := len(vulnerableEntries.(map[string]interface{}))
numPackages := len(auditedEntries)

var summaryHeader = []string{"Audited Count", "Vulnerable Count", "Build Version"}
var invalidHeader = []string{"Count", "Package", "Reason"}
var auditedHeader = []string{"Count", "Package", "Is Vulnerable", "Num Vulnerabilities", "Vulnerabilities"}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also maybe add these "can be updated" packages to the csv output too?? Maybe a column of "Latest non-vulnerable version" or something like that??

var summaryRow = []string{strconv.Itoa(packageCount), strconv.Itoa(numVulnerable), buildVersion}
var summaryRow = []string{strconv.Itoa(numPackages), strconv.Itoa(numVulnerable), buildVersion}

var buf bytes.Buffer
w := csv.NewWriter(&buf)
Expand Down Expand Up @@ -105,7 +104,7 @@ func (f CsvFormatter) Format(entry *Entry) ([]byte, error) {
auditEntry := auditedEntries[i-1]
if auditEntry.IsVulnerable() || !f.Quiet {
jsonVulns, _ := json.Marshal(auditEntry.Vulnerabilities)
if err = f.write(w, []string{"[" + strconv.Itoa(i) + "/" + strconv.Itoa(packageCount) + "]", auditEntry.Coordinates, strconv.FormatBool(auditEntry.IsVulnerable()), strconv.Itoa(len(auditEntry.Vulnerabilities)), string(jsonVulns)}); err != nil {
if err = f.write(w, []string{"[" + strconv.Itoa(i) + "/" + strconv.Itoa(numPackages) + "]", auditEntry.Coordinates, strconv.FormatBool(auditEntry.IsVulnerable()), strconv.Itoa(len(auditEntry.Vulnerabilities)), string(jsonVulns)}); err != nil {
return nil, err
}
}
Expand Down
33 changes: 19 additions & 14 deletions internal/cmd/iq.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,20 @@ package cmd
import (
"errors"
"fmt"
"io"
"os"

"github.com/mitchellh/go-homedir"
"github.com/sonatype-nexus-community/go-sona-types/configuration"
"github.com/sonatype-nexus-community/go-sona-types/iq"
ossIndexTypes "github.com/sonatype-nexus-community/go-sona-types/ossindex/types"
"github.com/sonatype-nexus-community/nancy/internal/customerrors"
"github.com/sonatype-nexus-community/nancy/internal/logger"
"github.com/sonatype-nexus-community/nancy/packages"
"github.com/sonatype-nexus-community/nancy/parse"
"github.com/sonatype-nexus-community/nancy/types"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"io"
"os"
)

type iqServerFactory interface {
Expand Down Expand Up @@ -108,10 +108,9 @@ func doIQ(cmd *cobra.Command, args []string) (err error) {

printHeader(!configOssi.Quiet)

var purls []string
purls, err = getPurls()
dependencies, err := getDependencies()

err = auditWithIQServer(purls)
err = auditWithIQServer(dependencies)
if err != nil {
if errExit, ok := err.(customerrors.ErrorExit); ok {
os.Exit(errExit.ExitCode)
Expand All @@ -124,12 +123,14 @@ func doIQ(cmd *cobra.Command, args []string) (err error) {
return
}

func getPurls() (purls []string, err error) {
func getDependencies() (dependencies map[string]types.Dependency, err error) {
if configOssi.Path != "" {
var invalidPurls []string
if purls, invalidPurls, err = getPurlsFromPath(configOssi.Path); err != nil {
dependencies, err = getPurlsFromPath(configOssi.Path)
if err != nil {
panic(err)
}

invalidCoordinates := convertInvalidPurlsToCoordinates(invalidPurls)
logLady.WithField("invalid", invalidCoordinates).Info("")
} else {
Expand All @@ -138,16 +139,14 @@ func getPurls() (purls []string, err error) {
panic(err)
}

mod := packages.Mod{}

mod.ProjectList, err = parse.GoListAgnostic(os.Stdin)
dependencies, err = parse.GoListAgnostic(os.Stdin)
if err != nil {
logLady.WithError(err).Error("unexpected error in iq cmd")
panic(err)
}
purls = mod.ExtractPurlsFromManifest()
}
return purls, err

return dependencies, err
}

const (
Expand Down Expand Up @@ -228,10 +227,16 @@ func initIQConfig() {
}
}

func auditWithIQServer(purls []string) error {
func auditWithIQServer(dependencies map[string]types.Dependency) error {
iqServer := iqCreator.create()

logLady.Debug("Sending purls to be Audited by IQ Server")

var purls []string
for k := range dependencies {
purls = append(purls, k)
}

// go-sona-types library now takes care of querying both ossi and iq with reformatted purls as needed (to v or not to v).
res, err := iqServer.AuditPackages(purls)
if err != nil {
Expand Down
Loading