From cf97d1ac56677a1a6c82fb5387669c5e35f62486 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Mon, 23 Dec 2024 19:11:32 +0500 Subject: [PATCH 1/4] fixed uri regex issue --- pkg/detectors/privacy/privacy.go | 3 ++- pkg/detectors/uri/uri.go | 2 +- pkg/detectors/uri/uri_test.go | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/privacy/privacy.go b/pkg/detectors/privacy/privacy.go index 30a73016a89c..fb62f6c478af 100644 --- a/pkg/detectors/privacy/privacy.go +++ b/pkg/detectors/privacy/privacy.go @@ -3,10 +3,11 @@ package privacy import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 111869ea9a8c..4ccec0ffe285 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -30,7 +30,7 @@ var _ interface { } = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\w-\.]{3,50}:([\w-\.]{3,50})@[-.%\w\/:]+\b`) // TODO: make local addr opt-out defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index 7c0762bb0434..26af4e330974 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -13,6 +13,7 @@ import ( var ( validPattern = "https://kaNydBSAodo87dsm9asuiSAFtsd7.com:1234@qYY3SylY7fHP" + validPattern2 = `

http://username:password@127.0.0.1

` invalidPattern = "https://kaNydBSAodo87dsm9asuiSAFtsd7.com.1234@qYY3SylY7fHP" keyword = "uri" ) @@ -30,6 +31,11 @@ func TestURI_Pattern(t *testing.T) { input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), want: []string{validPattern}, }, + { + name: "valid pattern - capture two outputs", + input: fmt.Sprintf("%s token = '%s'", keyword, validPattern2), + want: []string{"http://username:password@127.0.0.1", "http://username:password@127.0.0.1"}, + }, { name: "invalid pattern", input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), From ffbcb5f2df4cde1e4940d8f90fd9655ea0a40aaf Mon Sep 17 00:00:00 2001 From: kashif khan Date: Mon, 30 Dec 2024 12:12:51 +0500 Subject: [PATCH 2/4] extended password special char set --- pkg/detectors/uri/uri.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 4ccec0ffe285..a9ebc9b88169 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -30,7 +30,7 @@ var _ interface { } = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\w-\.]{3,50}:([\w-\.]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\w-\.]{3,50}:([\w-\.%$^&#@]{3,50})@[-.%\w\/:]+\b`) // TODO: make local addr opt-out defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses From 86441a98750e54ca600a02c8baef8b5bce61f823 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 31 Dec 2024 12:17:41 +0500 Subject: [PATCH 3/4] rebased --- pkg/detectors/uri/uri.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index a9ebc9b88169..42b695d9f1e4 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -30,7 +30,7 @@ var _ interface { } = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\w-\.]{3,50}:([\w-\.%$^&#@]{3,50})@[-.%\w\/:]+\b`) + keyPat = regexp.MustCompile(`\b(?:https?:\/\/)?[\w-\.$~!]{3,50}:([\w-\.%$^&#]{3,50})@[-.\w]+\b`) // TODO: make local addr opt-out defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses @@ -131,6 +131,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } } + results = append(results, r) } From a085517f72c1031e5a0f43ff9e0857ad393371da Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 31 Dec 2024 12:19:32 +0500 Subject: [PATCH 4/4] updated test case --- pkg/detectors/uri/uri_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index 26af4e330974..bdde2b9e8555 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -32,9 +32,9 @@ func TestURI_Pattern(t *testing.T) { want: []string{validPattern}, }, { - name: "valid pattern - capture two outputs", + name: "valid pattern - do not process duplicate", input: fmt.Sprintf("%s token = '%s'", keyword, validPattern2), - want: []string{"http://username:password@127.0.0.1", "http://username:password@127.0.0.1"}, + want: []string{"http://username:password@127.0.0.1"}, }, { name: "invalid pattern",