added rss functionality to milla, watchlists now allow you to give th…

…em a color so they can actually show you what they matched, bunch of other changes

.golangci.yml

@@ -27,3 +27,4 @@ linters-settings:
         - github.com/yuin/gluare
         - gitlab.com/megalithic-llc/gluasocket
         - github.com/layeh/gopher-json
+        - github.com/mmcdole/gofeed

README.md

@@ -175,62 +175,74 @@ ircChannels = ["#channel1", "#channel2"]
 
 Please note that the bot does not have to join a channel to be usable. One can simply query the bot directly as well.<br/>
 
-### databaseUser
+#### databaseUser
 
 Name of the database user.
 
-### databasePassword
+#### databasePassword
 
 Password for the database user.
 
-### databaseAddress
+#### databaseAddress
 
 Address of the database.
 
-### databaseName
+#### databaseName
 
 Name of the database.
 
-### ircProxy
+#### scrapeChannels
+
+List of channels that the bot will scrape into a database table. You can later on use these databases for the custom commands.<br/>
+
+#### ircProxy
 
 Determines which proxy to use to connect to the IRC network:
 
 ```
 ircProxy = "socks5://127.0.0.1:9050"
 ```
 
-### llmProxy
+#### llmProxy
 
 Determines which proxy to use to connect to the LLM endpoint:
 
 ```
 llmProxy = "socks5://127.0.0.1:9050"
 ```
 
-### ircdName
+#### ircdName
 
 Name of the milla instance, must be unique across all instances.
 
-### adminOnly
+#### adminOnly
 
 Milla will only answer if the nick is in the admin list.
 
-### webIRCGateway
+#### webIRCGateway
 
 webirc gateway to use.
 
-### webIRCHostname
+#### webIRCHostname
 
 webirc hostname to use.
 
-### webIRCPassword
+#### webIRCPassword
 
 webirc password to use.
 
-### webIRCAddress
+#### webIRCAddress
 
 webirc address to use.
 
+#### rssFile
+
+The file that contains the rss feeeds.
+
+#### channel
+
+The channel to send the rss feeds to.
+
 ### plugins
 
 A list of plugins to load:`plugins = ["./plugins/rss.lua", "./plugins/test.lua"]`
@@ -277,6 +289,32 @@ fgColor = 0
 bgColor = 28
 ```
 
+## RSS
+
+The rss file is self-explanatory. Here's an example:
+
+```json
+{
+  "feeds": [
+    {
+      "name": "one",
+      "url": "http://feeds.feedburner.com/crunchyroll/rss",
+      "proxy": "socks5://172.17.0.1:9007",
+      "userAgent": "Mozilla/5.0 (X11; U; Linux i686; pl; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1 (Ubuntu-edgy)",
+      "timeout": 10
+    },
+    {
+      "name": "two",
+      "url": "http://feeds.feedburner.com/crunchyroll/rss/anime",
+      "proxy": "socks5://172.17.0.1:9007",
+      "userAgent": "Mozilla/5.0 (X11; U; Linux i686; pl; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1 (Ubuntu-edgy)",
+      "timeout": 10
+    }
+  ],
+  "period": 3600
+}
+```
+
 ### Example Config File
 
 ```toml
@@ -311,10 +349,24 @@ skipTLSVerify = false
 useTLS = true
 plugins = ["/plugins/plugin1.lua", "/plugins/plugin2.lua"]
 adminOnly = false
+plugins = ["/plugins/ip.lua", "/plugins/urban.lua", "/plugins/remind.lua"]
+[ircd.devinet.watchlist.security]
+watchList = ["#securityfeeds"]
+watchFiles = ["/watchfiles/voidbox.list"]
+alertChannel = "#milla_alerts"
+[ircd.devinet.rss.manga]
+rssFile = "/rssfeeds/manga.json"
+channel = "#manga"
+[ircd.devinet.rss.anime]
+rssFile = "/rssfeeds/anime.json"
+channel = "#anime"
 [ircd.devinet.watchlist.security]
 watchList = ["#securityfeeds"]
 watchFiles = ["/watchfiles/voidbox.list"]
 alertChannel = "#milla_alerts"
+eventTypes = ["PRIVMSG"]
+fgColor = 0
+bgColor = 28
 
 [ircd.liberanet]
 ircServer = "irc.libera.chat"
@@ -343,12 +395,12 @@ out = true
 ircProxy = "socks5://127.0.0.1:9051"
 llmProxy = "http://127.0.0.1:8181"
 adminOnly = true
-[ircd.devinet_terra.customCommands.digest]
+[ircd.liberanet.customCommands.digest]
 sql = "select log from liberanet_milla_us_market_news order by log desc;"
 limit = 300
 context = ["you are a sentiment-analysis bot"]
 prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please provide the digest of the news for me."
-[ircd.devinet_terra.customCommands.summarize]
+[ircd.liberanet.customCommands.summarize]
 sql= "select log from liberanet_milla_us_market_news order by log desc;"
 limit= 300
 context = ["you are a sentiment-analysis bot"]
@@ -393,6 +445,14 @@ Load a plugin: `/load /plugins/rss.lua`
 
 Unload a plugin: `/unload /plugins/rss.lua`
 
+#### remind
+
+Pings the user after the given amount in seconds: `/remind 1200`
+
+#### roll
+
+Rolls a number between 1 and 6 if no arguments are given. With one argument it rolls a number between 1 and the given number. With two arguments it rolls a number between the two numbers: `/rool 10000 66666`
+
 ## Deploy
 
 ### Docker
@@ -768,6 +828,7 @@ isp: Cloudflare, Inc -- query: 1.1.1.1 -- status: success -- regionName: Queensl
 
 - Each lua plugin gets its own lua state and will run in a goroutine.<br/>
 - Lua plugins will not go through a proxy if they are not instructed to do so. If you are using the provided http module, you can set the proxy value before loading the http module as provided in the examples under `plugins`. The module will read and set the following environment variables in the order given:
+
   - `ALL_PROXY`
   - `HTTPS_PROXY`
   - `HTTP_PROXY`

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/kohkimakimoto/gluayaml v0.0.0-20160815032708-6fe413d49d73
 	github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf
 	github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e
+	github.com/mmcdole/gofeed v1.3.0
 	github.com/sashabaranov/go-openai v1.19.3
 	github.com/yuin/gluare v0.0.0-20170607022532-d7c94f1a80ed
 	github.com/yuin/gopher-lua v1.1.1
@@ -27,6 +28,8 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/longrunning v0.5.6 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
@@ -40,7 +43,11 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect

go.sum

@@ -13,6 +13,8 @@ cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXm
 cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/ailncode/gluaxmlpath v0.0.0-20161126153117-6ce478ecb4a6 h1:FM0WudTZ+xeiXPJcs+X1Zg8JXe4vlb9P2GZYqCYjZkk=
 github.com/ailncode/gluaxmlpath v0.0.0-20161126153117-6ce478ecb4a6/go.mod h1:Ti1AvV2KUYtHEBX7eYbdAGEfFyKz9+lHrJPcr79Vkng=
 github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink=
@@ -21,6 +23,8 @@ github.com/alecthomas/chroma/v2 v2.12.0 h1:Wh8qLEgMMsN7mgyG8/qIpegky2Hvzr4By6gEF
 github.com/alecthomas/chroma/v2 v2.12.0/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF0EpseFXdKMBw=
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 h1:rdWOzitWlNYeUsXmz+IQfa9NkGEq3gA/qQ3mOEqBU6o=
 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9/go.mod h1:X97UjDTXp+7bayQSFZk2hPvCTmTZIicUjZQRtkwgAKY=
@@ -69,6 +73,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
 github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -88,6 +93,8 @@ github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
 github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
 github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
 github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kohkimakimoto/gluayaml v0.0.0-20160815032708-6fe413d49d73 h1:e2UU76WBjv6W0QUuPHe2QfSAXLR1kouek1fcSUtnSrk=
 github.com/kohkimakimoto/gluayaml v0.0.0-20160815032708-6fe413d49d73/go.mod h1:I+YgUp/uc0hF+H4RuQjR+uzDJNjUz7Sm21e63UCLWWQ=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -98,6 +105,15 @@ github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:bg6J/5S/AeTz7
 github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:E/q28EyUVBgBQnONAVPIdwvEsv4Ve0vaCA9JWim4+3I=
 github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e h1:Y86mAFtJjS4P0atZ6QAKH88TV0ASQYJdIGWiOmJKoNY=
 github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e/go.mod h1:lgrnhcF8bg/Bd5HA5DOb4Z+uGqUqGnp4skr+J2GwVgI=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -147,6 +163,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
 golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -161,10 +178,14 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=

main.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"index/suffixarray"
 	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
@@ -222,6 +223,8 @@ func getHelpString() string {
 	helpString += "memstats - returns the memory status currently being used\n"
 	helpString += "load - loads a lua script\n"
 	helpString += "unload - unloads a lua script\n"
+	helpString += "remind - reminds you in a given amount of seconds\n"
+	helpString += "roll - rolls a dice. the number is between 1 and 6. One arg sets the upper limit. Two args sets the lower and upper limit in that order\n"
 
 	return helpString
 }
@@ -546,6 +549,71 @@ func runCommand(
 		}
 
 		appConfig.deleteLstate(args[1])
+	case "remind":
+		if len(args) < 2 {
+			client.Cmd.Reply(event, errNotEnoughArgs.Error())
+
+			break
+		}
+
+		seconds, err := strconv.Atoi(args[1])
+		if err != nil {
+			client.Cmd.Reply(event, errNotEnoughArgs.Error())
+
+			break
+		}
+
+		client.Cmd.Reply(event, "Ok, I'll remind you in "+args[1]+" seconds.")
+		time.Sleep(time.Duration(seconds) * time.Second)
+
+		client.Cmd.ReplyTo(event, " Ping!")
+	case "forget":
+
+		client.Cmd.Reply(event, "I no longer even know whether you're supposed to wear or drink a camel.'")
+	case "roll":
+		lowerLimit := 1
+		upperLimit := 6
+
+		if len(args) == 1 {
+		} else if len(args) == 2 {
+			argOne, err := strconv.Atoi(args[1])
+
+			if err != nil {
+				client.Cmd.Reply(event, errNotEnoughArgs.Error())
+
+				break
+			}
+
+			upperLimit = argOne
+		} else if len(args) == 3 {
+			argOne, err := strconv.Atoi(args[1])
+
+			if err != nil {
+				client.Cmd.Reply(event, errNotEnoughArgs.Error())
+
+				break
+			}
+
+			lowerLimit = argOne
+
+			argTwo, err := strconv.Atoi(args[2])
+
+			if err != nil {
+				client.Cmd.Reply(event, errNotEnoughArgs.Error())
+
+				break
+			}
+
+			upperLimit = argTwo
+		} else {
+			client.Cmd.Reply(event, errors.New("too many args").Error())
+
+			break
+		}
+
+		randomNumber := lowerLimit + rand.Intn(upperLimit-lowerLimit+1)
+
+		client.Cmd.ReplyTo(event, fmt.Sprint(randomNumber))
 	default:
 		_, ok := appConfig.LuaCommands[args[0]]
 		if !ok {
@@ -1262,6 +1330,12 @@ func runIRC(appConfig TomlConfig) {
 		go WatchListHandler(irc, appConfig)
 	}
 
+	if len(appConfig.Rss) > 0 {
+		irc.Handlers.AddBg(girc.CONNECTED, func(client *girc.Client, _ girc.Event) {
+			go runRSS(&appConfig, irc)
+		})
+	}
+
 	for {
 		var dialer proxy.Dialer