fixes #14, fixes #16, fixes #17, fixes #18, fixes #19, fixes #20

terminaldweller · May 15, 2024 · 006cb3a · 006cb3a
1 parent 108e204
commit 006cb3a
Show file tree

Hide file tree

Showing 7 changed files with 498 additions and 21 deletions.
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -35,6 +35,7 @@ jobs:
           context: .
           file: ./Dockerfile
           push: true
+          sbom: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           provenance: mode=max

diff --git a/.golangci.yml b/.golangci.yml
@@ -17,3 +17,4 @@ linters-settings:
         - github.com/lrstanley/girc
         - github.com/sashabaranov/go-openai
         - github.com/BurntSushi/toml
+        - github.com/jackc/pgx/v5/pgxpool
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # milla
 
-Milla is an IRC bot that sends things over to an LLM when you ask it questions and prints the answer with optional syntax-hilighting.<br/>
+Milla is an IRC bot that sends things over to an LLM when you ask it questions and prints the answer with optional syntax-highlighting.<br/>
 Currently Supported:
 
 - Ollama
@@ -39,7 +39,7 @@ The SASL username.
 
 #### ircSaslPass
 
-The SASL password for SASL plain authentication.
+The SASL password for SASL plain authentication. Can also be passed as and environment variable.
 
 #### ollamaEndpoint
 
@@ -77,7 +77,7 @@ Which LLM provider to use. The supported options are:
 
 #### apikey
 
-The apikey to use for the LLM provider.
+The apikey to use for the LLM provider. Can also be passed as and environment variable.
 
 #### ollamaSystem
 
@@ -89,7 +89,7 @@ The path to the client certificate to use for client cert authentication.
 
 #### serverPass
 
-The password to use for the IRC server the bot is trying to connect to if the server has a password.
+The password to use for the IRC server the bot is trying to connect to if the server has a password. Can also be passed as and environment variable.
 
 #### bind
 
@@ -169,6 +169,38 @@ List of channels for the bot to join when it connects to the server.
 ircChannels = ["#channel1", "#channel2"]
 ```
 
+### databaseUser
+
+Name of the database user. Can also be passed an an environment variable.
+
+### databasePassword
+
+Password for the database user. Can also be passed an an environment variable.
+
+### databaseAddress
+
+Address of the database. Can also be passed as and environment variable.
+
+### databaseName
+
+Name of the database. Can also be passed as and environment variable.
+
+### ircProxy
+
+Determines which proxy to use to connect to the irc network:
+
+```
+ircProxy = "socks5://127.0.0.1:9050"
+```
+
+### llmProxy
+
+Determines which proxy to use to connect to the LLM endpoint:
+
+```
+llmProxy = "socks5://127.0.0.1:9050"
+```
+
 ## Commands
 
 #### help
@@ -187,6 +219,20 @@ Get the value of all config options.
 
 Set a config option on the fly. Use the same name as the config file but capitalized.
 
+#### memstats
+
+Returns memory stats for milla.
+
+## Environment Variables
+
+- MILLA_SASL_PASSWORD
+- MILLA_SERVER_PASSWORD
+- MILLA_APIKEY
+- MILLA_DB_USER
+- MILLA_DB_PASSWORD
+- MILLA_DB_ADDRESS
+- MILLA_DB_NAME
+
 ## Proxy Support
 
 milla will read and use the `ALL_PROXY` environment variable.
@@ -200,6 +246,9 @@ ALL_PROXY=127.0.0.1:9050
 
 ## Deploy
 
+### Docker
+
+Images are automatically pushed to dockerhub. So you can get it from [there](https://hub.docker.com/r/terminaldweller/milla).
 An example docker compose file is provided in the repo under `docker-compose.yaml`.
 milla can be used with [gvisor](https://gvisor.dev/)'s docker runtime, `runsc`.
 
@@ -234,15 +283,141 @@ networks:
     driver: bridge
 ```
 
+### Public Message Storage
+
+milla can be configured to store all incoming public messages for future use in a postgres database. An example docker compose file is provided under `docker-compose-postgres.yaml`.<br/>
+
+```yaml
+services:
+  terra:
+    image: milla_distroless_vendored
+    build:
+      context: .
+      dockerfile: ./Dockerfile_distroless_vendored
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+    networks:
+      - terranet
+    user: 1000:1000
+    restart: unless-stopped
+    entrypoint: ["/usr/bin/milla"]
+    command: ["--config", "/config.toml"]
+    volumes:
+      - ./config-gpt.toml:/config.toml
+      - /etc/localtime:/etc/localtime:ro
+    cap_drop:
+      - ALL
+    environment:
+      - HTTPS_PROXY=http://172.17.0.1:8120
+      - https_proxy=http://172.17.0.1:8120
+      - HTTP_PROXY=http://172.17.0.1:8120
+      - http_proxy=http://172.17.0.1:8120
+  postgres:
+    image: postgres:16-alpine3.19
+    deploy:
+      resources:
+        limits:
+          memory: 4096M
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200m"
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:5455:5432/tcp"
+    volumes:
+      - terra_postgres_vault:/var/lib/postgresql/data
+      - ./scripts/:/docker-entrypoint-initdb.d/:ro
+    environment:
+      - POSTGRES_PASSWORD_FILE=/run/secrets/pg_pass_secret
+      - POSTGRES_USER_FILE=/run/secrets/pg_user_secret
+      - POSTGRES_INITDB_ARGS_FILE=/run/secrets/pg_initdb_args_secret
+      - POSTGRES_DB_FILE=/run/secrets/pg_db_secret
+    networks:
+      - terranet
+      - dbnet
+    secrets:
+      - pg_pass_secret
+      - pg_user_secret
+      - pg_initdb_args_secret
+      - pg_db_secret
+    runtime: runsc
+  pgadmin:
+    image: dpage/pgadmin4:8.6
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+    environment:
+      - PGADMIN_LISTEN_PORT=${PGADMIN_LISTEN_PORT:-5050}
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:[email protected]}
+      - PGADMIN_DEFAULT_PASSWORD_FILE=/run/secrets/pgadmin_pass
+      - PGADMIN_DISABLE_POSTFIX=${PGADMIN_DISABLE_POSTFIX:-YES}
+    ports:
+      - "127.0.0.1:5050:5050/tcp"
+    restart: unless-stopped
+    volumes:
+      - terra_pgadmin_vault:/var/lib/pgadmin
+    networks:
+      - dbnet
+    secrets:
+      - pgadmin_pass
+networks:
+  terranet:
+    driver: bridge
+  dbnet:
+volumes:
+  terra_postgres_vault:
+  terra_pgadmin_vault:
+secrets:
+  pg_pass_secret:
+    file: ./pg/pg_pass_secret
+  pg_user_secret:
+    file: ./pg/pg_user_secret
+  pg_initdb_args_secret:
+    file: ./pg/pg_initdb_args_secret
+  pg_db_secret:
+    file: ./pg/pg_db_secret
+  pgadmin_pass:
+    file: ./pgadmin/pgadmin_pass
+```
+
 The env vars `UID`and `GID`need to be defined or they can replaces by your host user's uid and gid.<br/>
 
-As a convinience, there is a a [distroless](https://github.com/GoogleContainerTools/distroless) dockerfile, `Dockerfile_distroless` also provided.<br/>
+As a convenience, there is a a [distroless](https://github.com/GoogleContainerTools/distroless) dockerfile, `Dockerfile_distroless` also provided.<br/>
 A vendored build of milla is available by first running `go mod vendor` and then using the provided Dockerfile, `Dockerfile_distroless_vendored`.<br/>
 
+### Build
+
+For a regular build:
+
+```sh
+go mod download
+go build
+```
+
+For a vendored build:
+
+```sh
+go mod vendor
+go build
+```
+
 ## Thanks
 
 - [girc](https://github.com/lrstanley/girc)
 - [chroma](https://github.com/alecthomas/chroma)
+- [pgx](https://github.com/jackc/pgx)
 - [ollama](https://github.com/ollama/ollama)
 
 ## Similar Projects

diff --git a/docker-compose-postgres.yaml b/docker-compose-postgres.yaml
@@ -0,0 +1,102 @@
+services:
+  terra:
+    image: milla_distroless_vendored
+    build:
+      context: .
+      dockerfile: ./Dockerfile_distroless_vendored
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+    networks:
+      - terranet
+    user: 1000:1000
+    restart: unless-stopped
+    entrypoint: ["/usr/bin/milla"]
+    command: ["--config", "/config.toml"]
+    volumes:
+      - ./config-gpt.toml:/config.toml
+      - /etc/localtime:/etc/localtime:ro
+    cap_drop:
+      - ALL
+    environment:
+      - HTTPS_PROXY=http://172.17.0.1:8120
+      - https_proxy=http://172.17.0.1:8120
+      - HTTP_PROXY=http://172.17.0.1:8120
+      - http_proxy=http://172.17.0.1:8120
+  postgres:
+    image: postgres:16-alpine3.19
+    deploy:
+      resources:
+        limits:
+          memory: 4096M
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200m"
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:5455:5432/tcp"
+    volumes:
+      - terra_postgres_vault:/var/lib/postgresql/data
+      - ./scripts/:/docker-entrypoint-initdb.d/:ro
+    environment:
+      - POSTGRES_PASSWORD_FILE=/run/secrets/pg_pass_secret
+      - POSTGRES_USER_FILE=/run/secrets/pg_user_secret
+      - POSTGRES_INITDB_ARGS_FILE=/run/secrets/pg_initdb_args_secret
+      - POSTGRES_DB_FILE=/run/secrets/pg_db_secret
+    networks:
+      - terranet
+      - dbnet
+    secrets:
+      - pg_pass_secret
+      - pg_user_secret
+      - pg_initdb_args_secret
+      - pg_db_secret
+    runtime: runsc
+  pgadmin:
+    image: dpage/pgadmin4:8.6
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+    environment:
+      - PGADMIN_LISTEN_PORT=${PGADMIN_LISTEN_PORT:-5050}
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:[email protected]}
+      - PGADMIN_DEFAULT_PASSWORD_FILE=/run/secrets/pgadmin_pass
+      - PGADMIN_DISABLE_POSTFIX=${PGADMIN_DISABLE_POSTFIX:-YES}
+    ports:
+      - "127.0.0.1:5050:5050/tcp"
+    restart: unless-stopped
+    volumes:
+      - terra_pgadmin_vault:/var/lib/pgadmin
+    networks:
+      - dbnet
+    secrets:
+      - pgadmin_pass
+networks:
+  terranet:
+    driver: bridge
+  dbnet:
+volumes:
+  terra_postgres_vault:
+  terra_pgadmin_vault:
+secrets:
+  pg_pass_secret:
+    file: ./pg/pg_pass_secret
+  pg_user_secret:
+    file: ./pg/pg_user_secret
+  pg_initdb_args_secret:
+    file: ./pg/pg_initdb_args_secret
+  pg_db_secret:
+    file: ./pg/pg_db_secret
+  pgadmin_pass:
+    file: ./pgadmin/pgadmin_pass
diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/alecthomas/chroma/v2 v2.12.0
 	github.com/google/generative-ai-go v0.11.2
+	github.com/jackc/pgx/v5 v5.5.5
 	github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e
 	github.com/sashabaranov/go-openai v1.19.3
 	golang.org/x/net v0.24.0
@@ -29,6 +30,9 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.4 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect