Add user choice UI

package.json

@@ -41,7 +41,7 @@
   },
   "devDependencies": {
     "@hono/node-server": "^1.8.2",
-    "@mfng/webpack-rsc": "^4.0.0",
+    "@mfng/webpack-rsc": "^4.0.1",
     "@swc/core": "^1.3.22",
     "@types/aws-lambda": "^8.10.136",
     "@types/node": "^20.11.26",

src/app/ai-state.ts

@@ -9,27 +9,34 @@ export type AIStateItem =
       readonly name: string;
     };
 
-export type UserInput =
-  | {
-      readonly action: 'message';
-      readonly content: string;
-    }
-  | {
-      readonly action: 'select-image';
-      readonly url: string;
-    };
+export type UserInputAction = 'choose-option' | 'message' | 'select-image';
+
+export interface UserInput {
+  readonly action: UserInputAction;
+  readonly content: string;
+}
 
 export function fromUserInput(userInput: UserInput): AIStateItem {
-  switch (userInput.action) {
+  const {action, content} = userInput;
+
+  if (action === `message`) {
+    return {role: `user`, content};
+  }
+
+  return {
+    role: `assistant`,
+    content: getAssistantStateContent(action, content),
+  };
+}
+
+function getAssistantStateContent(
+  action: 'choose-option' | 'select-image',
+  content: string,
+): string {
+  switch (action) {
+    case `choose-option`:
+      return `[user has chosen: ${content}]`;
     case `select-image`:
-      return {
-        role: `assistant`,
-        content: `[user wants to know more about the image ${userInput.url}. keep it short.]`,
-      };
-    case `message`:
-      return {
-        role: `user`,
-        content: userInput.content,
-      };
+      return `[user wants to know more about the image ${content}. keep it short.]`;
   }
 }

src/app/chat.tsx

@@ -49,6 +49,7 @@ export function Chat({children}: React.PropsWithChildren): React.ReactNode {
     ]);
 
     setInputValue(``);
+    document.body.scrollIntoView({block: `end`, behavior: `smooth`});
 
     startTransition(async () => {
       setOptimisticMessages((prevMessages) => [

src/app/google-image-search.ts

@@ -12,13 +12,13 @@ export const imageSearchParams = z.object({
     .string()
     .optional()
     .describe(
-      `Identifies a word or phrase that all documents in the search results must contain.`,
+      `Identifies a word or phrase that all search results must contain. Useful for artwork titles.`,
     ),
   excludeTerms: z
     .string()
     .optional()
     .describe(
-      `Identifies a word or phrase that should not appear in any documents in the search results.`,
+      `Identifies a word or phrase that should not appear in any search result.`,
     ),
   imgColorType: z
     .enum([`color`, `gray`, `mono`, `trans`])

src/app/image-selector.tsx

@@ -1,9 +1,7 @@
 'use client';
 
-import {useActions, useUIState} from 'ai/rsc';
 import * as React from 'react';
-import type {AI} from './ai.js';
-import {getErrorMessage} from './get-error-message.js';
+import {useSubmitUserMessage} from './use-submit-user-message.js';
 
 export type ImageSelectorProps = React.PropsWithChildren<{
   readonly url: string;
@@ -13,39 +11,14 @@ export function ImageSelector({
   children,
   url,
 }: ImageSelectorProps): React.ReactNode {
-  const [, setMessages] = useUIState<typeof AI>();
-  const {submitUserMessage} = useActions<typeof AI>();
-
-  const handleClick = async () => {
-    const optimisticMessageId = Date.now();
-
-    setMessages((prevMessages) => [
-      ...prevMessages,
-      {id: optimisticMessageId, role: `assistant`, display: <p>&hellip;</p>},
-    ]);
-
-    document.body.scrollIntoView({block: `end`, behavior: `smooth`});
-
-    try {
-      const message = await submitUserMessage({action: `select-image`, url});
-
-      setMessages((prevMessages) => [
-        ...prevMessages.filter(({id}) => id !== optimisticMessageId),
-        message,
-      ]);
-    } catch (error) {
-      console.error(error);
-      const errorMessage = getErrorMessage(error);
-
-      setMessages((prevMessages) => [
-        ...prevMessages.filter(({id}) => id !== optimisticMessageId),
-        {id: Date.now(), role: `error`, display: <p>{errorMessage}</p>},
-      ]);
-    }
-  };
+  const handleClick = useSubmitUserMessage(`select-image`, url);
 
   return (
-    <button className="cursor-pointer" type="button" onClick={handleClick}>
+    <button
+      className="cursor-pointer outline-cyan-500"
+      type="button"
+      onClick={handleClick}
+    >
       {children}
     </button>
   );

src/app/submit-user-message.tsx

@@ -12,6 +12,7 @@ import {createImageSet, serializeImageSets} from './image-set.js';
 import {LoadingIndicator} from './loading-indicator.js';
 import {Markdown} from './markdown.js';
 import {ProgressiveImage} from './progressive-image.js';
+import {UserChoiceButton} from './user-choice-button.js';
 
 const openai = new OpenAI({apiKey: process.env.OPENAI_API_KEY});
 
@@ -33,17 +34,27 @@ export async function submitUserMessage(
     messages: [
       {
         role: `system`,
-        content: `You are a chat assistant with a focus on images (photos, paintaings, cliparts, animated gifs, etc.).
+        content: `You are a chat assistant with a focus on images (photos, paintings, cliparts, animated gifs, etc.).
 
         A user might ask for images of any kind (only safe for work, though!) and you search for them and show them, using the input of the user for the various search parameters.
 
-        Use markdown in your messages if it improves how you can structure a response, highlight certain parts (especially the discussed subject's name should be strong), or to add links to other websites. Don't include images in markdown though, use the dedicated function instead.
+        Always prefer to show specific titled art pieces. If in doubt, just pick some that you know.
+
+        When it comes to art, try to show known images to the user with their title. Think about it step by step to come up with specific search queries. E.g. first select a style, then select an artist for that style, and then search for the specific painting (e.g. surrealism -> Dalí -> The Persistence of Memory -> query terms: ["Salvador Dalí", "The Persistence of Memory"]). Do this process a couple of times to come up with a diverse set of images in one single message.
+
+        Use markdown in your messages if it improves how you can structure a response, highlight certain parts (especially the discussed subject's name/title should be strong), or to add links to other websites.
+
+        Don't include images in markdown, use the dedicated tool instead.
 
         Never ask the user whether they want to see images of the discussed subject, always show them unprompted.
 
         Before showing images of a certain artist it might make sense to introduce them to the user first, with a couple of words.
 
         The user can also select an image if they want to know more about it.
+
+        When asking the user a question, you may present them with options to choose from.
+
+        Use the provided tools to show an interactive UI, along with your textual messages.
         `,
       },
       ...aiState.get(),
@@ -58,12 +69,74 @@ export async function submitUserMessage(
       return <Markdown text={content} />;
     },
     tools: {
+      get_choice_from_user: {
+        description: `Show a UI to let the user choose between several options`,
+        parameters: z.object({
+          intro: z
+            .string()
+            .describe(
+              `An introduction text that explains the options, or just poses the question if it wasn't asked before. Markdown allowed.`,
+            ),
+          options: z.array(
+            z.object({
+              id: z
+                .string()
+                .describe(
+                  `A short technical ID that's unique among all options.`,
+                ),
+              label: z.string()
+                .describe(`A short label for a UI element that the user can use to select the option.
+
+                  If you're asking a yes/no question, prefer verbose labels, e.g. "Yes, I want to see them"/"No, I'm good" over "yes"/"no".`),
+            }),
+          ),
+        }),
+        render({intro, options}) {
+          console.log(`get_choice_from_user`, options);
+
+          if (lastTextContent) {
+            aiState.update((prevAiState) => [
+              ...prevAiState,
+              {role: `assistant`, content: lastTextContent!},
+            ]);
+          }
+
+          aiState.done([
+            ...aiState.get(),
+            {
+              role: `function`,
+              name: `get_choice_from_user`,
+              content: JSON.stringify({options}),
+            },
+          ]);
+
+          return (
+            <div className="space-y-4">
+              {lastTextContent && (
+                <div>
+                  <Markdown text={lastTextContent} />
+                </div>
+              )}
+              <div>
+                <Markdown text={intro} />
+              </div>
+              <div className="mb-3 space-y-2 last:mb-0">
+                {options.map(({id, label}) => (
+                  <UserChoiceButton key={id} optionId={id}>
+                    {label}
+                  </UserChoiceButton>
+                ))}
+              </div>
+            </div>
+          );
+        },
+      },
       search_and_show_images: {
         description: `Search for images and show them.
 
-        When searching for artists, it might be helpful to put the medium into the query along with the artist's name (e.g. "paintings", "photos")
+        When searching for artists or styles (but not for specific art pieces), it might be helpful to put the medium into the query along with the artist's or styles' name (e.g. "paintings", "photos").
 
-        If the images to search for are distinctly different from each other (e.g. two different, titled paintings by the same artist), split it the search up into multiple search parameter sets.
+        If the images to search for are distinctly different from each other (e.g. two different, titled paintings by the same artist), split the search up into multiple search parameter sets.
 
         Select three images overall, unless the user asks for more or less. When searching for a specific artwork, select only one image.
 
@@ -75,33 +148,35 @@ export async function submitUserMessage(
             .describe(
               `A short text to show to the user while the image search is running.`,
             ),
-          searches: z.array(
-            z.object({
-              title: z
-                .string()
-                .describe(
-                  `The title can be used as an alt attribute or headline when showing images.`,
-                ),
-              notFoundMessage: z
-                .string()
-                .describe(
-                  `An error message to show to the user when no images were found.`,
-                ),
-              errorMessage: z
-                .string()
-                .describe(
-                  `An error message to show to the user when the search errored, most likely due to the search quota being exceeded for the current time period.`,
-                ),
-              searchParams: imageSearchParams,
-            }),
-          ),
+          searches: z
+            .array(
+              z.object({
+                title: z
+                  .string()
+                  .describe(
+                    `The title can be used as an alt attribute or headline when showing images.`,
+                  ),
+                notFoundMessage: z
+                  .string()
+                  .describe(
+                    `An error message to show to the user when no images were found.`,
+                  ),
+                errorMessage: z
+                  .string()
+                  .describe(
+                    `An error message to show to the user when the search errored, most likely due to the search quota being exceeded for the current time period.`,
+                  ),
+                searchParams: imageSearchParams,
+              }),
+            )
+            .describe(`Use multiple sets of search parameters if needed.`),
         }),
         async *render({loadingText, searches: searchParamsList}) {
-          console.log(`search params`, searchParamsList);
+          console.log(`search_and_show_images`, searchParamsList);
 
           const text = lastTextContent ? (
             <div>
-              <Markdown text={`${lastTextContent}`} />
+              <Markdown text={lastTextContent} />
             </div>
           ) : undefined;
 

src/app/use-submit-user-message.tsx

@@ -0,0 +1,41 @@
+import {useActions, useUIState} from 'ai/rsc';
+import * as React from 'react';
+import type {UserInputAction} from './ai-state.js';
+import type {AI} from './ai.js';
+import {getErrorMessage} from './get-error-message.js';
+
+export function useSubmitUserMessage(
+  action: UserInputAction,
+  content: string,
+): () => void {
+  const [, setMessages] = useUIState<typeof AI>();
+  const {submitUserMessage} = useActions<typeof AI>();
+
+  return React.useCallback(async () => {
+    const optimisticMessageId = Date.now();
+
+    setMessages((prevMessages) => [
+      ...prevMessages,
+      {id: optimisticMessageId, role: `assistant`, display: <p>&hellip;</p>},
+    ]);
+
+    document.body.scrollIntoView({block: `end`, behavior: `smooth`});
+
+    try {
+      const message = await submitUserMessage({action, content});
+
+      setMessages((prevMessages) => [
+        ...prevMessages.filter(({id}) => id !== optimisticMessageId),
+        message,
+      ]);
+    } catch (error) {
+      console.error(error);
+      const errorMessage = getErrorMessage(error);
+
+      setMessages((prevMessages) => [
+        ...prevMessages.filter(({id}) => id !== optimisticMessageId),
+        {id: Date.now(), role: `error`, display: <p>{errorMessage}</p>},
+      ]);
+    }
+  }, [submitUserMessage, action, content]);
+}