WIP

vintasoftware · Jun 21, 2024 · e34f4ee · e34f4ee
1 parent a3d4dd4
commit e34f4ee
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 14 deletions.
diff --git a/django_ai_assistant/api/schemas.py b/django_ai_assistant/api/schemas.py
@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Union
 
 from django.utils import timezone
 
@@ -29,7 +30,7 @@ class ThreadSchemaIn(Schema):
 
 class ThreadMessagesSchemaIn(Schema):
     assistant_id: str
-    content: str
+    content: Union[str, list[Union[str, dict]]]
 
 
 class ThreadMessageTypeEnum(str, Enum):
@@ -44,4 +45,4 @@ class ThreadMessageTypeEnum(str, Enum):
 class ThreadMessagesSchemaOut(Schema):
     id: str  # noqa: A003
     type: ThreadMessageTypeEnum  # noqa: A003
-    content: str
+    content: Union[str, list[Union[str, dict]]]
diff --git a/django_ai_assistant/helpers/use_cases.py b/django_ai_assistant/helpers/use_cases.py
@@ -75,8 +75,37 @@ def create_message(
 
     # TODO: Check if we can separate the message creation from the chain invoke
     assistant = assistant_cls(user=user, request=request)
+    print("\n\ncontent", content, "\n\n")
+    # assistant_message = assistant.invoke(
+    #     {"input": content},
+    #     thread_id=thread.id,
+    # )
     assistant_message = assistant.invoke(
-        {"input": content},
+        # {
+        #     "input": HumanMessage(
+        #         content=[
+        #             content,
+        #             "describe the weather",
+        #             # {
+        #             #     "type": "image_url",
+        #             #     "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/c/c8/Altja_j%C3%B5gi_Lahemaal.jpg/320px-Altja_j%C3%B5gi_Lahemaal.jpg",
+        #             # },
+        #         ],
+        #     )
+        # },
+        # [HumanMessage(content=content), HumanMessage(content="describe the weather")],
+        [
+            HumanMessage(
+                content=[
+                    content,
+                    "describe the weather",
+                    # {
+                    #     "type": "image_url",
+                    #     "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/c/c8/Altja_j%C3%B5gi_Lahemaal.jpg/320px-Altja_j%C3%B5gi_Lahemaal.jpg",
+                    # },
+                ],
+            )
+        ],
         thread_id=thread.id,
     )
     return assistant_message

diff --git a/example/assets/js/components/Chat/Chat.tsx b/example/assets/js/components/Chat/Chat.tsx
@@ -55,6 +55,8 @@ function ChatMessage({
     </Tooltip>
   );
 
+  console.log({ message });
+
   return (
     <Group
       gap="lg"
@@ -79,7 +81,11 @@ function ChatMessage({
         bg="var(--mantine-color-gray-0)"
       >
         <Group gap="md" justify="space-between" align="top">
-          <Markdown className={classes.mdMessage}>{message.content}</Markdown>
+          <Markdown className={classes.mdMessage}>
+            {Array.isArray(message.content)
+              ? message.content[0]
+              : message.content}
+          </Markdown>
         </Group>
       </Paper>
 

diff --git a/frontend/openapi_schema.json b/frontend/openapi_schema.json
@@ -434,8 +434,25 @@
             "$ref": "#/components/schemas/ThreadMessageTypeEnum"
           },
           "content": {
-            "title": "Content",
-            "type": "string"
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "items": {
+                  "anyOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object"
+                    }
+                  ]
+                },
+                "type": "array"
+              }
+            ],
+            "title": "Content"
           }
         },
         "required": [
@@ -453,8 +470,25 @@
             "type": "string"
           },
           "content": {
-            "title": "Content",
-            "type": "string"
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "items": {
+                  "anyOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object"
+                    }
+                  ]
+                },
+                "type": "array"
+              }
+            ],
+            "title": "Content"
           }
         },
         "required": [

diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts
@@ -84,8 +84,25 @@ export const $ThreadMessagesSchemaOut = {
             '$ref': '#/components/schemas/ThreadMessageTypeEnum'
         },
         content: {
-            title: 'Content',
-            type: 'string'
+            anyOf: [
+                {
+                    type: 'string'
+                },
+                {
+                    items: {
+                        anyOf: [
+                            {
+                                type: 'string'
+                            },
+                            {
+                                type: 'object'
+                            }
+                        ]
+                    },
+                    type: 'array'
+                }
+            ],
+            title: 'Content'
         }
     },
     required: ['id', 'type', 'content'],
@@ -100,8 +117,25 @@ export const $ThreadMessagesSchemaIn = {
             type: 'string'
         },
         content: {
-            title: 'Content',
-            type: 'string'
+            anyOf: [
+                {
+                    type: 'string'
+                },
+                {
+                    items: {
+                        anyOf: [
+                            {
+                                type: 'string'
+                            },
+                            {
+                                type: 'object'
+                            }
+                        ]
+                    },
+                    type: 'array'
+                }
+            ],
+            title: 'Content'
         }
     },
     required: ['assistant_id', 'content'],

diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts
@@ -21,12 +21,16 @@ export type ThreadMessageTypeEnum = 'human' | 'ai' | 'generic' | 'system' | 'fun
 export type ThreadMessagesSchemaOut = {
     id: string;
     type: ThreadMessageTypeEnum;
-    content: string;
+    content: string | Array<(string | {
+    [key: string]: unknown;
+})>;
 };
 
 export type ThreadMessagesSchemaIn = {
     assistant_id: string;
-    content: string;
+    content: string | Array<(string | {
+    [key: string]: unknown;
+})>;
 };
 
 export type DjangoAiAssistantListAssistantsResponse = Array<AssistantSchema>;