Merge pull request #41 from glitchedgitz/dev2024

Url parsing fixed
glitchedgitz · Apr 8, 2024 · acdcac4 · acdcac4
2 parents 3748721 + 9749596
commit acdcac4
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 2 deletions.
diff --git a/test_file/file1.txt → test/file1.txt b/test_file/file1.txt → test/file1.txt
diff --git a/test_file/file2.txt → test/file2.txt b/test_file/file2.txt → test/file2.txt
diff --git a/test_file/test.txt → test/test.txt b/test_file/test.txt → test/test.txt
diff --git a/test/urls.txt b/test/urls.txt
diff --git a/v2/pkg/methods/urls.go b/v2/pkg/methods/urls.go
@@ -5,6 +5,7 @@ import (
 	"net"
 	"net/url"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"golang.org/x/net/publicsuffix"
@@ -113,16 +114,31 @@ func (m *Methods) UrlAllDir(u *url.URL, array *[]string) {
 func (m *Methods) AnalyzeURLs(urls []string, fn func(*url.URL, *[]string), array *[]string) {
 
 	for _, s := range urls {
-		u, err := url.Parse(s)
+		if !strings.HasPrefix(s, "http://") && !strings.HasPrefix(s, "https://") {
+			s = "http://" + s
+		}
+		sanitizedURL := sanitizeURL(s)
+		u, err := url.Parse(sanitizedURL)
 		if err != nil {
-			log.Println("Err: AnalyseURLs in url " + s)
+			log.Println("Err: AnalyseURLs in url ", err)
 			continue
 		}
 
 		fn(u, array)
 	}
 }
 
+// Function to sanitize the URL string
+func sanitizeURL(s string) string {
+	// Regular expression to match any characters outside the valid ASCII range
+	controlCharsRegex := regexp.MustCompile(`[^ -~]`)
+
+	// Replace any characters outside the valid ASCII range with an empty string
+	sanitizedURL := controlCharsRegex.ReplaceAllString(s, "")
+
+	return sanitizedURL
+}
+
 func (m *Methods) init() {
 	log.SetFlags(0)
 }