From ac118a3eb0fb0943341731c76b1b6fba092348ce Mon Sep 17 00:00:00 2001 From: Tami Date: Fri, 26 Apr 2024 01:34:57 +0000 Subject: [PATCH 1/4] Added Groq Support - groq api integration now fits well into oi flow, no errors. Though final answers are halucinated rather than actual output. Seems to plan, write code, but not execute. --- interpreter/core/llm/llm.py | 39 ++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/interpreter/core/llm/llm.py b/interpreter/core/llm/llm.py index a539e59a8c..47b1f26762 100644 --- a/interpreter/core/llm/llm.py +++ b/interpreter/core/llm/llm.py @@ -1,4 +1,7 @@ import litellm +from groq import Groq +groq_client = [None] + import tokentrim as tt from ...terminal_interface.utils.display_markdown_message import ( @@ -10,6 +13,7 @@ litellm.suppress_debug_info = True import time +import os class Llm: @@ -26,6 +30,7 @@ def __init__(self, interpreter): # Settings self.model = "gpt-4-turbo" + self.model = "groq/mixtral-8x7b-32768" # can now use models from groq. `export GROQ_API_KEY="your-key-here")` or use --model self.temperature = 0 self.supports_vision = False self.supports_functions = None # Will try to auto-detect @@ -210,7 +215,39 @@ def fixed_litellm_completions(**params): # Run completion first_error = None try: - yield from litellm.completion(**params) + # print(f"!!!!!1 {params}\n"*10) + def source(**params): + '''Get Completions Using LiteLLM''' + yield from litellm.completion(**params) + + if "model" in params and "groq/" in params['model']: + def groq_complete(**params): + if groq_client[0] is None: + groq_client[0] = Groq( + # This is the default and can be omitted + api_key=os.environ.get("GROQ_API_KEY"), + timeout = 2, + max_retries = 3 + ) + res = groq_client[0].chat.completions.create( + messages = params['messages'], + model=params['model'].split("groq/")[1], + ).choices[0].message.content + print("@@@@@@@@@\n",res,'@@@@@@@@@@@') + return res + + def s(**params): + '''Get Completions Using Groq''' + params['stream'] = False # To keep things simple for now, and groq is super fast anyway + word_by_word = False + if word_by_word: + for chunk in groq_complete(**params).split(" "): + yield {"choices":[{"delta":{"type": "message", "content": chunk+" "}}]} + else: + for whole in [groq_complete(**params)]: + yield {"choices":[{"delta":{"type": "message", "content": whole}}]} + source = s + yield from source(**params) except Exception as e: # Store the first error first_error = e diff --git a/pyproject.toml b/pyproject.toml index 79809ccdc7..f6474da6ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ matplotlib = "^3.8.2" toml = "^0.10.2" posthog = "^3.1.0" tiktoken = "^0.6.0" +groq = "^4.3.0" #Optional dependencies From 553cb51d0c347830699426c26324354a7ff07e76 Mon Sep 17 00:00:00 2001 From: Tami Date: Fri, 26 Apr 2024 01:47:42 +0000 Subject: [PATCH 2/4] Used black and isort (not sure if looks better but following CONTRIBUTING.md + minor cleanup --- interpreter/core/llm/llm.py | 62 +++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/interpreter/core/llm/llm.py b/interpreter/core/llm/llm.py index 47b1f26762..35bcdae472 100644 --- a/interpreter/core/llm/llm.py +++ b/interpreter/core/llm/llm.py @@ -1,5 +1,6 @@ import litellm from groq import Groq + groq_client = [None] import tokentrim as tt @@ -12,8 +13,8 @@ from .utils.convert_to_openai_messages import convert_to_openai_messages litellm.suppress_debug_info = True -import time import os +import time class Llm: @@ -30,7 +31,7 @@ def __init__(self, interpreter): # Settings self.model = "gpt-4-turbo" - self.model = "groq/mixtral-8x7b-32768" # can now use models from groq. `export GROQ_API_KEY="your-key-here")` or use --model + self.model = "groq/mixtral-8x7b-32768" # can now use models from groq. `export GROQ_API_KEY="your-key-here")` or use --model self.temperature = 0 self.supports_vision = False self.supports_functions = None # Will try to auto-detect @@ -72,7 +73,7 @@ def run(self, messages): self.supports_functions = False except: self.supports_functions = False - + # Trim image messages if they're there if self.supports_vision: image_messages = [msg for msg in messages if msg["type"] == "image"] @@ -215,37 +216,52 @@ def fixed_litellm_completions(**params): # Run completion first_error = None try: - # print(f"!!!!!1 {params}\n"*10) def source(**params): - '''Get Completions Using LiteLLM''' + """Get Completions Using LiteLLM""" yield from litellm.completion(**params) - - if "model" in params and "groq/" in params['model']: + + if "model" in params and "groq/" in params["model"]: + def groq_complete(**params): if groq_client[0] is None: - groq_client[0] = Groq( - # This is the default and can be omitted - api_key=os.environ.get("GROQ_API_KEY"), - timeout = 2, - max_retries = 3 - ) - res = groq_client[0].chat.completions.create( - messages = params['messages'], - model=params['model'].split("groq/")[1], - ).choices[0].message.content - print("@@@@@@@@@\n",res,'@@@@@@@@@@@') + groq_client[0] = Groq( + # This is the default and can be omitted + api_key=os.environ.get("GROQ_API_KEY"), + timeout=2, + max_retries=3, + ) + res = ( + groq_client[0] + .chat.completions.create( + messages=params["messages"], + model=params["model"].split("groq/")[1], + ) + .choices[0] + .message.content + ) return res - + def s(**params): - '''Get Completions Using Groq''' - params['stream'] = False # To keep things simple for now, and groq is super fast anyway + """Get Completions Using Groq""" + params["stream"] = ( + False # To keep things simple for now, and groq is super fast anyway + ) word_by_word = False if word_by_word: for chunk in groq_complete(**params).split(" "): - yield {"choices":[{"delta":{"type": "message", "content": chunk+" "}}]} + yield { + "choices": [ + {"delta": {"type": "message", "content": chunk + " "}} + ] + } else: for whole in [groq_complete(**params)]: - yield {"choices":[{"delta":{"type": "message", "content": whole}}]} + yield { + "choices": [ + {"delta": {"type": "message", "content": whole}} + ] + } + source = s yield from source(**params) except Exception as e: From 66e87843c57500e0406de7ccd96ca3534965b42e Mon Sep 17 00:00:00 2001 From: Tami Date: Fri, 26 Apr 2024 02:02:10 +0000 Subject: [PATCH 3/4] Added proper usage docs --- docs/language-models/hosted-models/groq.mdx | 52 +++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/language-models/hosted-models/groq.mdx diff --git a/docs/language-models/hosted-models/groq.mdx b/docs/language-models/hosted-models/groq.mdx new file mode 100644 index 0000000000..aee02e27db --- /dev/null +++ b/docs/language-models/hosted-models/groq.mdx @@ -0,0 +1,52 @@ +--- +title: Groq +--- + +To use Open Interpreter with a model from Groq, simply run: + + + +```bash Terminal +interpreter --model groq/mixtral-8x7b-32768 +``` + +```python Python +from interpreter import interpreter + +interpreter.llm.model = "groq/mixtral-8x7b-32768" +interpreter.chat() +``` + + + +# Supported Models + +We support any model on [Groq's models page:](https://console.groq.com/docs/models) + + + +```bash Terminal +interpreter --model groq/mixtral-8x7b-32768 +interpreter --model groq/llama3-8b-8192 +interpreter --model groq/llama3-70b-8192 +interpreter --model groq/gemma-7b-it +``` + +```python Python +interpreter.llm.model = "groq/mixtral-8x7b-32768" +interpreter.llm.model = "groq/llama3-8b-8192" +interpreter.llm.model = "groq/llama3-70b-8192" +interpreter.llm.model = "groq/gemma-7b-it" +``` + + + +# Required Environment Variables + +Run `export GROQ_API_KEY=''` or place it in your rc file and re-source +Set the following environment variables [(click here to learn how)](https://chat.openai.com/share/1062cdd8-62a1-4aa8-8ec9-eca45645971a) to use these models. + +| Environment Variable | Description | Where to Find | +| -------------------- | ---------------------------------------------------- | ------------------------------------------------------------------- | +| `GROQ_API_KEY` | The API key for authenticating to Groq's services. | [Groq Account Page](https://console.groq.com/keys) | + From 33300f1a3e67f26739040552ee558d01e81dae17 Mon Sep 17 00:00:00 2001 From: Tami Date: Tue, 7 May 2024 21:51:40 +0300 Subject: [PATCH 4/4] Included workaround in docs incase anyone runs into this issue --- docs/language-models/hosted-models/groq.mdx | 26 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/language-models/hosted-models/groq.mdx b/docs/language-models/hosted-models/groq.mdx index aee02e27db..f0c151e46b 100644 --- a/docs/language-models/hosted-models/groq.mdx +++ b/docs/language-models/hosted-models/groq.mdx @@ -7,13 +7,34 @@ To use Open Interpreter with a model from Groq, simply run: ```bash Terminal -interpreter --model groq/mixtral-8x7b-32768 +interpreter --model groq/llama3-8b-8192 ``` ```python Python from interpreter import interpreter -interpreter.llm.model = "groq/mixtral-8x7b-32768" +interpreter.llm.model = "groq/llama3-8b-8192" +interpreter.llm.api_key = '' +interpreter.chat() +``` + + + +If you are having any issues when passing the `--model`, try adding the `--api_base`: + + + +```bash Terminal +interpreter --api_base "https://api.groq.com/openai/v1" --model groq/llama3-8b-8192 --api_key $GROQ_API_KEY +``` + +```python Python +from interpreter import interpreter + +interpreter.llm.model = "groq/llama3-8b-8192" +interpreter.llm.api_key = '' +interpreter.llm.api_base = "https://api.groq.com/openai/v1" +interpreter.llm.context_window = 32000 interpreter.chat() ``` @@ -49,4 +70,3 @@ Set the following environment variables [(click here to learn how)](https://chat | Environment Variable | Description | Where to Find | | -------------------- | ---------------------------------------------------- | ------------------------------------------------------------------- | | `GROQ_API_KEY` | The API key for authenticating to Groq's services. | [Groq Account Page](https://console.groq.com/keys) | -