diff --git a/apis/crawler/crawler.go b/apis/crawler/crawler.go index 82f48e7..6cda8ec 100644 --- a/apis/crawler/crawler.go +++ b/apis/crawler/crawler.go @@ -109,8 +109,12 @@ func openURL(myURL string) (TableRow, error) { results.LastURL = hooks.Truncate(myURL, 200) results.LastStatusCode = int16(resp.StatusCode) urlStr, _ := url.Parse(myURL) - myIP, _ := net.LookupIP(urlStr.Hostname()) - results.IP = hooks.Truncate(myIP[0].String(), 30) + myIP, err := net.LookupIP(urlStr.Hostname()) + if err != nil { + hooks.LogIfNeeded(manager.Logger, fmt.Sprintf("No IP found for %v: %v", results.LastURL , err), manager.LogLevel, hooks.LogError) + } else { + results.IP = hooks.Truncate(myIP[0].String(), 30) + } rCodes = append(rCodes, fmt.Sprintf("%d", resp.StatusCode)) if resp.StatusCode/100 == 3 { diff --git a/apis/ssllabs/Certificates.sql b/apis/ssllabs/Certificates.sql index 9b21c2d..ada5ed4 100644 --- a/apis/ssllabs/Certificates.sql +++ b/apis/ssllabs/Certificates.sql @@ -9,9 +9,10 @@ CREATE TABLE [CertificatesV10]( [Issues] [smallint] NULL, [KeyStrength] [smallint] NULL, [DebianInsecure] [bit] NULL, - [NotBefore] [bigint] NULL, - [NotAfter] [bigint] NULL, [NextThumbprint] [nchar](40) NULL, + [ValidFrom] [Datetime] NULL, + [ValidTo] [DateTime] NULL, + [AltNames] nvarchar(MAX) NULL, CONSTRAINT [PK_CertificatesV10] PRIMARY KEY CLUSTERED ( [Thumbprint] ASC diff --git a/apis/ssllabs/SSLLabs.sql b/apis/ssllabs/SSLLabs.sql index 44259d9..26dbf27 100644 --- a/apis/ssllabs/SSLLabs.sql +++ b/apis/ssllabs/SSLLabs.sql @@ -5,8 +5,8 @@ CREATE TABLE [SSLLabsV10]( [DomainReachable] [tinyint] NOT NULL, [ScanStatus] [tinyint] NOT NULL, [IP] [nvarchar](30) NULL, - [StartTime] [bigint] NULL, - [TestTime] [bigint] NULL, + [StartTime] [DateTime] NULL, + [TestTime] [DateTime] NULL, [Grade] [nvarchar](2) NULL, [GradeTrustIgnored] [nvarchar](2) NULL, [FutureGrade] [nvarchar](2) NULL, diff --git a/apis/ssllabs/ssllabsscan.go b/apis/ssllabs/ssllabsscan.go index 92241cb..fab0628 100644 --- a/apis/ssllabs/ssllabsscan.go +++ b/apis/ssllabs/ssllabsscan.go @@ -141,8 +141,8 @@ func (e LabsErrorResponse) Error() string { type TableRow struct { IP string - StartTime int64 - TestTime int64 + StartTime string + TestTime string Grade string GradeTrustIgnored string FutureGrade string @@ -907,15 +907,16 @@ func makeCertificateRows(report *LabsReport) []*hooks.CertificateRow { row.Issues = int16(cert.Issues) row.KeyStrength = int16(cert.KeyStrength) row.DebianInsecure = cert.KeyKnownDebianInsecure - row.NotAfter = cert.NotAfter - row.NotBefore = cert.NotBefore + row.ValidTo = time.Unix(cert.NotAfter/1000, 0).Format("2006-01-02 15:04:05") + row.ValidFrom = time.Unix(cert.NotBefore/1000, 0).Format("2006-01-02 15:04:05") + row.AltNames = strings.Join(cert.AltNames, ", ") if i+1 < chainLength { row.NextThumbprint = sql.NullString{ String: hooks.Truncate(report.Certs[i+1].Sha1Hash, 40), Valid: true, } } - + res = append([]*hooks.CertificateRow{row}, res...) } return res @@ -932,8 +933,8 @@ func makeSSLLabsRow(report *LabsReport) *TableRow { details := endpoint.Details row := &TableRow{} row.IP = hooks.Truncate(endpoint.IpAddress, 15) - row.StartTime = report.StartTime - row.TestTime = report.TestTime + row.StartTime = time.Unix(report.StartTime/1000, 0).Format("2006-01-02 15:04:05") + row.TestTime = time.Unix(report.TestTime/1000, 0).Format("2006-01-02 15:04:05") row.Grade = hooks.Truncate(endpoint.Grade, 2) row.GradeTrustIgnored = hooks.Truncate(endpoint.GradeTrustIgnored, 2) row.FutureGrade = hooks.Truncate(endpoint.FutureGrade, 2) diff --git a/backend/sqlWriter.go b/backend/sqlWriter.go index 8ddffda..22af8c3 100644 --- a/backend/sqlWriter.go +++ b/backend/sqlWriter.go @@ -386,14 +386,14 @@ func SaveCertificates(rows []*hooks.CertificateRow, table string) error { IF NOT EXISTS (SELECT * FROM %[1]v WHERE Thumbprint = ?) BEGIN INSERT INTO %[1]v - (Thumbprint, SerialNumber, Subject, Issuer, SigAlg, RevocationStatus, Issues, KeyStrength, DebianInsecure, NotBefore, NotAfter, NextThumbprint) + (Thumbprint, SerialNumber, Subject, Issuer, SigAlg, RevocationStatus, Issues, KeyStrength, DebianInsecure, NextThumbprint, ValidFrom, ValidTo, AltNames) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) END ELSE BEGIN UPDATE %[1]v - SET Issues = ? + SET Issues = ?, ValidFrom = ?, ValidTo = ?, AltNames = ? WHERE Thumbprint = ? END ` @@ -401,7 +401,7 @@ ELSE _, err = globalDatabase.ExecContext(ctx, fmt.Sprintf(query, table), row.Thumbprint, row.Thumbprint, row.SerialNumber, row.Subject, row.Issuer, row.SigAlg, row.RevocationStatus, row.Issues, - row.KeyStrength, row.DebianInsecure, row.NotBefore, row.NotAfter, row.NextThumbprint, row.Issues, row.Thumbprint) + row.KeyStrength, row.DebianInsecure, row.NextThumbprint, row.ValidFrom, row.ValidTo, row.AltNames, row.Issues, row.ValidFrom, row.ValidTo, row.AltNames, row.Thumbprint) if err != nil { return err } diff --git a/hooks/hooks.go b/hooks/hooks.go index 38304b3..7e41508 100644 --- a/hooks/hooks.go +++ b/hooks/hooks.go @@ -162,9 +162,10 @@ type CertificateRow struct { Issues int16 KeyStrength int16 DebianInsecure bool - NotBefore int64 - NotAfter int64 NextThumbprint sql.NullString + ValidFrom string + ValidTo string + AltNames string } type DomainsRowMetaInfo struct {