Merge pull request #177 from vulncheck-oss/sqlitedb

#156 Initial implementation of sqlite3 integration

cli/commandline.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/vulncheck-oss/go-exploit/c2"
 	"github.com/vulncheck-oss/go-exploit/config"
+	"github.com/vulncheck-oss/go-exploit/db"
 	"github.com/vulncheck-oss/go-exploit/output"
 	"github.com/vulncheck-oss/go-exploit/protocol"
 )
@@ -401,6 +402,27 @@ func sslFlags(conf *config.Config) {
 	flag.BoolVar(&conf.DetermineSSL, "a", false, "Automatically determine if the remote target uses SSL")
 }
 
+// Allow the user to provide the `-db` command to share content across exploits and limit the size of the HTTP cache.
+func dbFlags(conf *config.Config) {
+	flag.StringVar(&conf.DBName, "db", "", "An SQLite3 DB to share target data across")
+	flag.IntVar(&db.GlobalHTTPRespCacheLimit, "cacheMax", 10*(1024*1024), "The maximum size of an HTTP response that can be cached")
+}
+
+// handlue generic sounding c2 flags.
+func c2Flags(conf *config.Config) {
+	// flags unique to remote code execution
+	flag.IntVar(&conf.Bport, "bport", 0, "The port to attach the bind shell to")
+
+	// checks if the default values have been changed in the exploit directly
+	if conf.C2Timeout != 30 && conf.C2Timeout != 0 {
+		flag.IntVar(&conf.C2Timeout, "t", conf.C2Timeout, "The number of seconds to listen for reverse shells.")
+	} else {
+		flag.IntVar(&conf.C2Timeout, "t", 30, "The number of seconds to listen for reverse shells.")
+	}
+
+	flag.BoolVar(&conf.ThirdPartyC2Server, "o", false, "Indicates if the reverse shell should be caught by an outside program (nc, openssl)")
+}
+
 // Parses the command line arguments used by RCE exploits.
 func CodeExecutionCmdLineParse(conf *config.Config) bool {
 	var rhosts string
@@ -411,22 +433,14 @@ func CodeExecutionCmdLineParse(conf *config.Config) bool {
 	var exploitLogLevel string
 	var proxy string
 
+	dbFlags(conf)
 	proxyFlags(&proxy)
 	loggingFlags(&logFile, &frameworkLogLevel, &exploitLogLevel)
 	remoteHostFlags(conf, &rhosts, &rhostsFile, &rports)
 	localHostFlags(conf)
 	exploitFunctionality(conf)
 	sslFlags(conf)
-
-	// flags unique to remote code execution
-	flag.IntVar(&conf.Bport, "bport", 0, "The port to attach the bind shell to")
-	// checks if the default values have been changed in the exploit directly
-	if conf.C2Timeout != 30 && conf.C2Timeout != 0 {
-		flag.IntVar(&conf.C2Timeout, "t", conf.C2Timeout, "The number of seconds to listen for reverse shells.")
-	} else {
-		flag.IntVar(&conf.C2Timeout, "t", 30, "The number of seconds to listen for reverse shells.")
-	}
-	flag.BoolVar(&conf.ThirdPartyC2Server, "o", false, "Indicates if the reverse shell should be caught by an outside program (nc, openssl)")
+	c2Flags(conf)
 
 	// c2 selection. defaults to the implementations first supported value
 	var c2Selection string
@@ -513,6 +527,7 @@ func InformationDisclosureCmdLineParse(conf *config.Config) bool {
 	var exploitLogLevel string
 	var proxy string
 
+	dbFlags(conf)
 	proxyFlags(&proxy)
 	loggingFlags(&logFile, &frameworkLogLevel, &exploitLogLevel)
 	remoteHostFlags(conf, &rhosts, &rhostsFile, &rports)
@@ -548,6 +563,7 @@ func WebShellCmdLineParse(conf *config.Config) bool {
 	var exploitLogLevel string
 	var proxy string
 
+	dbFlags(conf)
 	proxyFlags(&proxy)
 	loggingFlags(&logFile, &frameworkLogLevel, &exploitLogLevel)
 	remoteHostFlags(conf, &rhosts, &rhostsFile, &rports)

config/config.go

@@ -71,6 +71,8 @@ type Config struct {
 	C2Timeout int
 	// Indicates if the c2 server will be handled elsewhere
 	ThirdPartyC2Server bool
+	// The database we are working with
+	DBName string
 }
 
 func New(extype ExploitType, supportedC2 []c2.Impl, product string, cve string, defaultPort int) *Config {

db/create.go

@@ -0,0 +1,156 @@
+// SQLite Caching and Cross-Exploit Database
+//
+// The db package contains the logic to handle a user provided SQLite DB in order
+// to store results and cache HTTP responses. This has a few useful benefits:
+//
+//  1. When scanning with hundreds of go-exploit implementations, the user significantly
+//     cuts down on network requests (therefore speeding up scanning), both from the
+//     verified results being cached (you only have to verify a target is Confluence once)
+//     and from the cached HTTP responses.
+//  2. The result is a useful asset database containing IP, port, installed software, and
+//     versions.
+//  3. The database can be reused with a go-exploit and generate no network traffic (assuming
+//     you aren't doing the exploitation stage). That is very interesting when, for example,
+//     you wrote a version scanner for CVE-2024-31982, scanned a customer host that was patched,
+//     but then CVE-2024-31983 is released the next day. You can essentially rescan the cached
+//     version of their system with your new CVE scanner.
+//
+// Mostly this package should be totally transparent to users of the framework. The only direct
+// interface, currently, should be calls to HTTPGetCache.
+package db
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/vulncheck-oss/go-exploit/output"
+	// pure go sqlite3 driver.
+	_ "modernc.org/sqlite"
+)
+
+// GlobalSQLHandle is a handle to the SQLite DB for handling cross-exploit data sharing.
+var GlobalSQLHandle *sql.DB
+
+// GlobalHTTPRespCacheLimit is the maximum size of an HTTP body that we will attempt to cache.
+var GlobalHTTPRespCacheLimit int
+
+const (
+	schemaVersion = "1.0.0"
+
+	metadataTable     = `CREATE TABLE IF NOT EXISTS metadata (created INTEGER NOT NULL,schema_version TEXT NOT NULL);`
+	initMetadataTable = `INSERT INTO metadata (created, schema_version) VALUES (?, ?)`
+	checkMetadata     = `SELECT name FROM sqlite_master WHERE type ='table' = ? AND name='metadata`
+	getSchemaVersion  = `SELECT schema_version FROM metadata;`
+
+	verifiedTable = `CREATE TABLE IF NOT EXISTS verified(` +
+		`id INTEGER PRIMARY KEY AUTOINCREMENT,` +
+		`created INTEGER NOT NULL,` +
+		`software_name TEXT NOT NULL,` +
+		`installed INTEGER NOT NULL CHECK (installed IN (0, 1)),` +
+		`version TEXT NOT NULL,` +
+		`rhost TEXT NOT NULL,` +
+		`rport INTEGER NOT NULL CHECK (rport >= 0 AND rport <= 65535));`
+
+	httpCacheTable = `CREATE TABLE IF NOT EXISTS http_cache(` +
+		`id INTEGER PRIMARY KEY AUTOINCREMENT,` +
+		`created INTEGER NOT NULL,` +
+		`rhost TEXT NOT NULL,` +
+		`rport INTEGER NOT NULL CHECK (rport >= 0 AND rport <= 65535),` +
+		`uri TEXT NOT NULL,` +
+		`data BLOB NOT NULL);`
+)
+
+func InitializeDB(name string) bool {
+	GlobalSQLHandle = nil
+	if len(name) == 0 {
+		return true
+	}
+
+	handle, err := sql.Open("sqlite", name)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	if checkMetadataExists(handle) && !checkSchemaVersion(handle) {
+		return false
+	} else if !createMetadataTable(handle) {
+		return false
+	}
+
+	if !createVerifiedSoftwareTable(handle) ||
+		!createHTTPCacheTable(handle) {
+		return false
+	}
+
+	GlobalSQLHandle = handle
+
+	return true
+}
+
+func checkMetadataExists(handle *sql.DB) bool {
+	name := ""
+	err := handle.QueryRow(checkMetadata).Scan(&name)
+
+	return err == nil
+}
+
+func createMetadataTable(handle *sql.DB) bool {
+	_, err := handle.Exec(metadataTable)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	_, err = handle.Exec(initMetadataTable, time.Now().Unix(), schemaVersion)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	return true
+}
+
+func checkSchemaVersion(handle *sql.DB) bool {
+	version := ""
+	err := handle.QueryRow(getSchemaVersion).Scan(&version)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	if version != schemaVersion {
+		output.PrintFrameworkError("Incompatible schema versions")
+
+		return false
+	}
+
+	return true
+}
+
+func createVerifiedSoftwareTable(handle *sql.DB) bool {
+	_, err := handle.Exec(verifiedTable)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	return true
+}
+
+func createHTTPCacheTable(handle *sql.DB) bool {
+	// create the cache table
+	_, err := handle.Exec(httpCacheTable)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	return true
+}

db/get.go

@@ -0,0 +1,31 @@
+package db
+
+const (
+	getCacheData = `SELECT data FROM http_cache WHERE rhost = ? AND rport = ? AND uri = ?`
+	getInstalled = `SELECT installed FROM verified WHERE software_name = ? AND rhost = ? AND rport = ?`
+)
+
+// Look for an HTTP response in the db cache.
+func GetHTTPResponse(rhost string, rport int, path string) (string, bool) {
+	if GlobalSQLHandle == nil {
+		return "", false
+	}
+
+	var retVal []byte
+	err := GlobalSQLHandle.QueryRow(getCacheData, rhost, rport, path).Scan(&retVal)
+
+	return string(retVal), err == nil
+}
+
+// Check the database to see if the target has been scanned for specific software. If so, return the result (so we don't do it again)
+// Return is <db-value>,<ok>.
+func GetVerified(product string, rhost string, rport int) (bool, bool) {
+	if GlobalSQLHandle == nil {
+		return false, false
+	}
+
+	retVal := false
+	err := GlobalSQLHandle.QueryRow(getInstalled, product, rhost, rport).Scan(&retVal)
+
+	return retVal, err == nil
+}

db/update.go

@@ -0,0 +1,62 @@
+package db
+
+import (
+	"time"
+
+	"github.com/vulncheck-oss/go-exploit/output"
+	// pure go sqlite3 driver.
+	_ "modernc.org/sqlite"
+)
+
+const (
+	verifiedUpsert = `INSERT INTO verified (id, created, software_name, installed, version, rhost, rport)` +
+		`VALUES ((SELECT id FROM verified WHERE rhost = ? AND rport = ? AND software_name = ?), ?, ?, ?, ?, ?, ?)` +
+		`ON CONFLICT(id) DO UPDATE SET ` +
+		`software_name = excluded.software_name,` +
+		`installed = excluded.installed,` +
+		`rhost = excluded.rhost,` +
+		`rport = excluded.rport,` +
+		`version = excluded.version;`
+
+	cacheUpsert = `INSERT INTO http_cache (id, created, rhost, rport, uri, data)` +
+		`VALUES ((SELECT id FROM http_cache WHERE rhost = ? AND rport = ? AND uri = ?), ?, ?, ?, ?, ?)` +
+		`ON CONFLICT(id) DO UPDATE SET ` +
+		`rhost = excluded.rhost,` +
+		`rport = excluded.rport,` +
+		`uri = excluded.uri,` +
+		`data = excluded.data;`
+)
+
+func UpdateVerified(software string, installed bool, version string, rhost string, rport int) bool {
+	if GlobalSQLHandle == nil {
+		return true
+	}
+
+	_, err := GlobalSQLHandle.Exec(verifiedUpsert, rhost, rport, software, time.Now().Unix(), software, installed, version, rhost, rport)
+	if err != nil {
+		output.PrintFrameworkError(err.Error())
+
+		return false
+	}
+
+	return true
+}
+
+// Attempt to cache the provided HTTP httpResp in the database.
+func CacheHTTPResponse(rhost string, rport int, path string, httpResp []byte) {
+	if GlobalSQLHandle == nil {
+		return
+	}
+
+	// only cache up to a user configurable size
+	if len(httpResp) > GlobalHTTPRespCacheLimit {
+		return
+	}
+
+	_, err := GlobalSQLHandle.Exec(cacheUpsert, rhost, rport, path, time.Now().Unix(), rhost, rport, path, httpResp)
+	if err != nil {
+		output.PrintfFrameworkError("Error during caching: %s", err.Error())
+
+		return
+	}
+}

docs/db.md

@@ -0,0 +1,50 @@
+# Database Usage
+
+go-exploit supports the use of an SQLite3 database in order to facilite cross-exploit communication and HTTP caching. This is optional, but can greatly improve the performance of large scale scanning. 
+
+To use this feature, use the `-db` command line option. The provided file can be empty or a database created during previous runs of go-exploit. In the example below, we use the option `-db vc.db` to use the non-existent file `vc.db`:
+
+```console
+albinolobster@mournland:~/initial-access/feed/cve-2024-31982$ ls vc.db
+ls: cannot access 'vc.db': No such file or directory
+albinolobster@mournland:~/initial-access/feed/cve-2024-31982$ ./build/cve-2024-31982_linux-arm64 -v -c -rhost 10.9.49.29 -rport 8080 -db vc.db -fll TRACE
+time=2024-06-26T15:54:18.902-04:00 level=DEBUG msg="Using the HTTP User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+time=2024-06-26T15:54:18.911-04:00 level=STATUS msg="Starting target" index=0 host=10.9.49.29 port=8080 ssl=false "ssl auto"=false
+time=2024-06-26T15:54:18.911-04:00 level=STATUS msg="Validating XWiki target" host=10.9.49.29 port=8080
+time=2024-06-26T15:54:19.068-04:00 level=SUCCESS msg="Target verification succeeded!" host=10.9.49.29 port=8080 verified=true
+time=2024-06-26T15:54:19.068-04:00 level=STATUS msg="Running a version check on the remote target" host=10.9.49.29 port=8080
+time=2024-06-26T15:54:19.068-04:00 level=TRACE msg="HTTP cache hit: http://10.9.49.29:8080/"
+time=2024-06-26T15:54:19.069-04:00 level=VERSION msg="The reported version is 14.10.7" host=10.9.49.29 port=8080 version=14.10.7
+time=2024-06-26T15:54:19.070-04:00 level=SUCCESS msg="The target appears to be a vulnerable version!" host=10.9.49.29 port=8080 vulnerable=yes
+albinolobster@mournland:~/initial-access/feed/cve-2024-31982$ ls vc.db 
+vc.db
+```
+
+In the TRACE output we can see that there is an HTTP cache hit during version scanning, this is because the first request in target verfication was cached. If we run the exploit again we will see more TRACE output:
+
+```console
+albinolobster@mournland:~/initial-access/feed/cve-2024-31982$ ./build/cve-2024-31982_linux-arm64 -v -c -rhost 10.9.49.29 -rport 8080 -db vc.db -fll TRACE
+time=2024-06-26T15:55:57.566-04:00 level=DEBUG msg="Using the HTTP User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+time=2024-06-26T15:55:57.570-04:00 level=STATUS msg="Starting target" index=0 host=10.9.49.29 port=8080 ssl=false "ssl auto"=false
+time=2024-06-26T15:55:57.570-04:00 level=STATUS msg="Validating XWiki target" host=10.9.49.29 port=8080
+time=2024-06-26T15:55:57.570-04:00 level=TRACE msg="Verified software cache hit" result=true
+time=2024-06-26T15:55:57.570-04:00 level=SUCCESS msg="Target verification succeeded!" host=10.9.49.29 port=8080 verified=true
+time=2024-06-26T15:55:57.570-04:00 level=STATUS msg="Running a version check on the remote target" host=10.9.49.29 port=8080
+time=2024-06-26T15:55:57.570-04:00 level=TRACE msg="HTTP cache hit: http://10.9.49.29:8080/"
+time=2024-06-26T15:55:57.570-04:00 level=VERSION msg="The reported version is 14.10.7" host=10.9.49.29 port=8080 version=14.10.7
+```
+
+Note the first TRACE this time is for `Verified software cache hit`. That is because the result of the previous run was saved in the database, so this exploit knows that 10.9.49.29:8080 is XWiki.
+
+```console
+albinolobster@mournland:~/initial-access/feed/cve-2024-31982$ sqlite3 vc.db 
+SQLite version 3.31.1 2020-01-27 19:55:54
+Enter ".help" for usage hints.
+sqlite> select * from verified;
+1|1719431659|XWiki|1|14.10.7|10.9.49.29|8080
+sqlite> 
+```
+
+In fact, on this second run, no network traffic was generated. This has a variety of useful applications including improved speed, asset database generation, easy to create test databases, and "scanless" scans.
+
+In order to utilize the DB, implementing exploits must use the HTTPGetCache API call.