Skip to content

Commit

Permalink
Support for Realtime API with Azure OpenAI (#848)
Browse files Browse the repository at this point in the history
Co-authored-by: Théo Monnom <[email protected]>
davidzhao and theomonnom authored Oct 8, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 3553810 commit a264627
Showing 6 changed files with 316 additions and 81 deletions.
5 changes: 5 additions & 0 deletions .changeset/proud-items-fail.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"livekit-plugins-openai": patch
---

Support for Realtime API with Azure OpenAI
24 changes: 21 additions & 3 deletions examples/multimodal_agent.py
Original file line number Diff line number Diff line change
@@ -44,12 +44,30 @@ async def get_weather(
return f"The weather in {location} is {weather_data}."
else:
raise Exception(
"Failed to get weather data, status code: {response.status}"
f"Failed to get weather data, status code: {response.status}"
)

await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
participant = await ctx.wait_for_participant()

assistant = multimodal.MultimodalAgent(
# to use Microsoft Azure, uncomment the following lines
# agent = multimodal.MultimodalAgent(
# model=openai.realtime.RealtimeModel.with_azure(
# azure_deployment="<model-deployment>",
# azure_endpoint="wss://<endpoint>.openai.azure.com/", # or AZURE_OPENAI_ENDPOINT
# api_key="<api-key>", # or AZURE_OPENAI_API_KEY
# api_version="2024-10-01-preview", # or OPENAI_API_VERSION
# voice="alloy",
# temperature=0.8,
# instructions="You are a helpful assistant",
# turn_detection=openai.realtime.ServerVadOptions(
# threshold=0.6, prefix_padding_ms=200, silence_duration_ms=500
# ),
# ),
# fnc_ctx=fnc_ctx,
# )

agent = multimodal.MultimodalAgent(
model=openai.realtime.RealtimeModel(
voice="alloy",
temperature=0.8,
@@ -60,7 +78,7 @@ async def get_weather(
),
fnc_ctx=fnc_ctx,
)
assistant.start(ctx.room)
agent.start(ctx.room, participant)


if __name__ == "__main__":
2 changes: 1 addition & 1 deletion livekit-agents/livekit/agents/llm/chat_context.py
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ def create(
if len(images) == 0:
return ChatMessage(role=role, content=text)
else:
content: list[str | ChatImage] = []
content: list[ChatContent] = []
if text:
content.append(text)

Original file line number Diff line number Diff line change
@@ -90,6 +90,7 @@ def __init__(
self._started, self._closed = False, False

self._update_state_task: asyncio.Task | None = None
self._http_session: aiohttp.ClientSession | None = None

@property
def vad(self) -> vad.VAD | None:
Original file line number Diff line number Diff line change
@@ -192,7 +192,8 @@ class SessionUpdateData(TypedDict):
tools: list[FunctionTool]
tool_choice: ToolChoice
temperature: float
max_response_output_tokens: int | Literal["inf"]
# microsoft does not support inf, but accepts None
max_response_output_tokens: int | Literal["inf"] | None

class SessionUpdate(TypedDict):
event_id: NotRequired[str]

Large diffs are not rendered by default.

0 comments on commit a264627

Please sign in to comment.