From 66eec626ab3bdb90c670eb1b63ac85f0302ccdb0 Mon Sep 17 00:00:00 2001
From: Leon Jacobs <leonja511@gmail.com>
Date: Thu, 19 Sep 2024 15:22:16 +0200
Subject: [PATCH] (fix) allow file parser to parse <host>:<port> targets

---
 Makefile                 |  6 ++-
 pkg/readers/file.go      |  8 ++--
 pkg/readers/file_test.go | 99 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 109 insertions(+), 4 deletions(-)
 create mode 100644 pkg/readers/file_test.go

diff --git a/Makefile b/Makefile
index 09cb312a..322f82b3 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ PLATFORMS := darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 linux/arm windows
 CGO := CGO_ENABLED=0
 
 # Default target
-default: clean frontend build integrity
+default: clean test frontend build integrity
 
 # Clean up build artifacts
 clean:
@@ -29,6 +29,10 @@ frontend: check-npm
 check-npm:
 	@command -v npm >/dev/null 2>&1 || { echo >&2 "npm is not installed. Please install npm first."; exit 1; }
 
+# Run any tests
+test:
+	@echo "Running tests..."
+	go test ./...
 
 # Build for all platforms
 build: $(PLATFORMS)
diff --git a/pkg/readers/file.go b/pkg/readers/file.go
index 1c611d48..e568dda8 100644
--- a/pkg/readers/file.go
+++ b/pkg/readers/file.go
@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/sensepost/gowitness/internal/islazy"
-	"github.com/sensepost/gowitness/pkg/log"
 )
 
 // FileReader is a reader that expects a file with targets that
@@ -80,6 +79,11 @@ func (fr *FileReader) Read(ch chan<- string) error {
 // If any ports configuration exists, those will also be added as candidates.
 func (fr *FileReader) urlsFor(candidate string, ports []int) []string {
 	var urls []string
+	// check if we got a scheme, add
+	hasScheme := strings.Contains(candidate, "://")
+	if !hasScheme {
+		candidate = "http://" + candidate
+	}
 
 	parsedURL, err := url.Parse(candidate)
 	if err != nil {
@@ -87,7 +91,6 @@ func (fr *FileReader) urlsFor(candidate string, ports []int) []string {
 		return urls
 	}
 
-	hasScheme := parsedURL.Scheme != ""
 	hasPort := parsedURL.Port() != ""
 	hostname := parsedURL.Hostname()
 
@@ -107,7 +110,6 @@ func (fr *FileReader) urlsFor(candidate string, ports []int) []string {
 
 		// at this point if hostname is still "", then just skip it entirely
 		if hostname == "" {
-			log.Debug("could not parse candidate to something usable", "candidate", candidate)
 			return urls
 		}
 	}
diff --git a/pkg/readers/file_test.go b/pkg/readers/file_test.go
new file mode 100644
index 00000000..06c649c5
--- /dev/null
+++ b/pkg/readers/file_test.go
@@ -0,0 +1,99 @@
+package readers
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestUrlsFor(t *testing.T) {
+	fr := FileReader{
+		Options: &FileReaderOptions{},
+	}
+
+	tests := []struct {
+		name      string
+		candidate string
+		ports     []int
+		want      []string
+	}{
+		{
+			name:      "Test with IP",
+			candidate: "192.168.1.1",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"http://192.168.1.1:80",
+				"http://192.168.1.1:443",
+				"http://192.168.1.1:8443",
+				"https://192.168.1.1:80",
+				"https://192.168.1.1:443",
+				"https://192.168.1.1:8443",
+			},
+		},
+		{
+			name:      "Test with IP and port",
+			candidate: "192.168.1.1:8080",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"http://192.168.1.1:8080",
+				"https://192.168.1.1:8080",
+			},
+		},
+		{
+			name:      "Test with scheme, IP and port",
+			candidate: "http://192.168.1.1:8080",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"http://192.168.1.1:8080",
+			},
+		},
+		{
+			name:      "Test with scheme and IP",
+			candidate: "https://192.168.1.1",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"https://192.168.1.1:80",
+				"https://192.168.1.1:443",
+				"https://192.168.1.1:8443",
+			},
+		},
+		{
+			name:      "Test with IP and path",
+			candidate: "192.168.1.1/path",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"http://192.168.1.1:80/path",
+				"http://192.168.1.1:443/path",
+				"http://192.168.1.1:8443/path",
+				"https://192.168.1.1:80/path",
+				"https://192.168.1.1:443/path",
+				"https://192.168.1.1:8443/path",
+			},
+		},
+		{
+			name:      "Test with scheme, IP, port and path",
+			candidate: "http://192.168.1.1:8080/path",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"http://192.168.1.1:8080/path",
+			},
+		},
+		{
+			name:      "Test with IP, port and path",
+			candidate: "192.168.1.1:8080/path",
+			ports:     []int{80, 443, 8443},
+			want: []string{
+				"http://192.168.1.1:8080/path",
+				"https://192.168.1.1:8080/path",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := fr.urlsFor(tt.candidate, tt.ports)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("urlsFor() =>\n\nhave: %v\nwant %v", got, tt.want)
+			}
+		})
+	}
+}