forked from microsoft/semantic-kernel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net: Fix function calling integration tests (microsoft#8948)
### Motivation, Context and Description Some of the function-calling integration tests were failing occasionally due to various reasons not related to the function-calling functionality in SK. For example, the AI model could decide to call the same function twice instead of once in one out of ten integration test runs. Additionally, Azure may have activated the content management policy by default, causing a few integration tests to fail with the error: "Error: Exception while invoking function. HTTP 400 (content_filter), Parameter: prompt. The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry." Lastly, some of the integration tests were disabled to unblock the latest release because the AI model started to call functions that were not related to the prompt but were advertised to the model. This PR fixes the intermittently failing tests and re-enables the disabled ones.
- Loading branch information
1 parent
be90d23
commit 8924bdc
Showing
6 changed files
with
80 additions
and
90 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -81,7 +81,20 @@ public async Task CanAutoInvokeKernelFunctionsStreamingAsync() | |
| public async Task CanAutoInvokeKernelFunctionsWithComplexTypeParametersAsync() | ||
| { | ||
| // Arrange | ||
| var kernel = this.CreateAndInitializeKernel(importHelperPlugin: true); | ||
| var kernel = this.CreateAndInitializeKernel(); | ||
| kernel.ImportPluginFromFunctions("HelperFunctions", | ||
| [ | ||
| kernel.CreateFunctionFromMethod((WeatherParameters parameters) => | ||
| { | ||
| if (parameters.City.Name.Equals("Dublin", StringComparison.OrdinalIgnoreCase) && | ||
| (parameters.City.Country.Equals("Ireland", StringComparison.OrdinalIgnoreCase) || parameters.City.Country.Equals("IE", StringComparison.OrdinalIgnoreCase))) | ||
| { | ||
| return Task.FromResult(42.8); // 42.8 Fahrenheit. | ||
| } | ||
|
|
||
| throw new NotSupportedException($"Weather in {parameters.City.Name} ({parameters.City.Country}) is not supported."); | ||
| }, "Get_Current_Temperature", "Get current temperature."), | ||
| ]); | ||
|
|
||
| AzureOpenAIPromptExecutionSettings settings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; | ||
|
|
||
|
|
@@ -129,10 +142,19 @@ public async Task CanAutoInvokeKernelFunctionsWithEnumTypeParametersAsync() | |
| public async Task CanAutoInvokeKernelFunctionFromPromptAsync() | ||
| { | ||
| // Arrange | ||
| var invokedFunctions = new List<string>(); | ||
|
|
||
| var filter = new FakeFunctionFilter(async (context, next) => | ||
| { | ||
| invokedFunctions.Add(context.Function.Name); | ||
| await next(context); | ||
| }); | ||
|
|
||
| var kernel = this.CreateAndInitializeKernel(); | ||
| kernel.FunctionInvocationFilters.Add(filter); | ||
|
|
||
| var promptFunction = KernelFunctionFactory.CreateFromPrompt( | ||
| "Your role is always to return this text - 'A Game-Changer for the Transportation Industry'. Don't ask for more details or context.", | ||
| "Hey LLM, give me one news title that's hot off the press!", | ||
| functionName: "FindLatestNews", | ||
| description: "Searches for the latest news."); | ||
|
|
||
|
|
@@ -144,21 +166,30 @@ public async Task CanAutoInvokeKernelFunctionFromPromptAsync() | |
| AzureOpenAIPromptExecutionSettings settings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; | ||
|
|
||
| // Act | ||
| var result = await kernel.InvokePromptAsync("Show me the latest news as they are.", new(settings)); | ||
| var result = await kernel.InvokePromptAsync("Show me the latest news.", new(settings)); | ||
|
|
||
| // Assert | ||
| Assert.NotNull(result); | ||
| Assert.Contains("Transportation", result.GetValue<string>(), StringComparison.InvariantCultureIgnoreCase); | ||
| Assert.Contains(invokedFunctions, functionName => functionName.Contains("InvokePromptAsync")); | ||
| Assert.Contains(invokedFunctions, functionName => functionName.Contains("FindLatestNews")); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task CanAutoInvokeKernelFunctionFromPromptStreamingAsync() | ||
| { | ||
| // Arrange | ||
| var invokedFunctions = new List<string>(); | ||
|
|
||
| var filter = new FakeFunctionFilter(async (context, next) => | ||
| { | ||
| invokedFunctions.Add(context.Function.Name); | ||
| await next(context); | ||
| }); | ||
|
|
||
| var kernel = this.CreateAndInitializeKernel(); | ||
| kernel.FunctionInvocationFilters.Add(filter); | ||
|
|
||
| var promptFunction = KernelFunctionFactory.CreateFromPrompt( | ||
| "Your role is always to return this text - 'A Game-Changer for the Transportation Industry'. Don't ask for more details or context.", | ||
| "Hey LLM, give me one news title that's hot off the press!", | ||
| functionName: "FindLatestNews", | ||
| description: "Searches for the latest news."); | ||
|
|
||
|
|
@@ -170,20 +201,14 @@ public async Task CanAutoInvokeKernelFunctionFromPromptStreamingAsync() | |
| AzureOpenAIPromptExecutionSettings settings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; | ||
|
|
||
| // Act | ||
| var streamingResult = kernel.InvokePromptStreamingAsync("Show me the latest news as they are.", new(settings)); | ||
|
|
||
| var builder = new StringBuilder(); | ||
|
|
||
| var streamingResult = kernel.InvokePromptStreamingAsync("Show me the latest news.", new(settings)); | ||
| await foreach (var update in streamingResult) | ||
| { | ||
| builder.Append(update.ToString()); | ||
| } | ||
|
|
||
| var result = builder.ToString(); | ||
|
|
||
| // Assert | ||
| Assert.NotNull(result); | ||
| Assert.Contains("Transportation", result, StringComparison.InvariantCultureIgnoreCase); | ||
| Assert.Contains(invokedFunctions, functionName => functionName.Contains("InvokePromptStreamingAsync")); | ||
| Assert.Contains(invokedFunctions, functionName => functionName.Contains("FindLatestNews")); | ||
| } | ||
|
|
||
| [Fact] | ||
|
|
@@ -482,7 +507,7 @@ public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForAutoFu | |
| Assert.NotNull(getWeatherForCityFunctionCallResult.Result); | ||
| } | ||
|
|
||
| [Fact(Skip = "Weather in Boston (USA) is not supported.")] | ||
| [Fact] | ||
| public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForManualFunctionCallingForStreamingAsync() | ||
| { | ||
| // Arrange | ||
|
|
@@ -784,10 +809,13 @@ public async Task ItShouldSupportOldFunctionCallingModelSerializedIntoChatHistor | |
| string? emailBody = null, emailRecipient = null; | ||
|
|
||
| var kernel = this.CreateAndInitializeKernel(importHelperPlugin: true); | ||
| kernel.ImportPluginFromFunctions("EmailPlugin", [KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail")]); | ||
| kernel.ImportPluginFromFunctions("EmailPlugin", [ | ||
| KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail"), | ||
| KernelFunctionFactory.CreateFromMethod(() => "[email protected]", "GetMyEmail") | ||
| ]); | ||
|
|
||
| // The deserialized chat history contains a list of function calls and the final answer to the question regarding the color of the sky in Boston. | ||
| chatHistory.AddUserMessage("Send the exact answer to my email: [email protected]"); | ||
| chatHistory.AddUserMessage("Send the exact answer to my email."); | ||
|
|
||
| var settings = new AzureOpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; | ||
|
|
||
|
|
@@ -820,10 +848,13 @@ public async Task ItShouldSupportNewFunctionCallingModelSerializedIntoChatHistor | |
| string? emailBody = null, emailRecipient = null; | ||
|
|
||
| var kernel = this.CreateAndInitializeKernel(importHelperPlugin: true); | ||
| kernel.ImportPluginFromFunctions("EmailPlugin", [KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail")]); | ||
| kernel.ImportPluginFromFunctions("EmailPlugin", [ | ||
| KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail"), | ||
| KernelFunctionFactory.CreateFromMethod(() => "[email protected]", "GetMyEmail") | ||
| ]); | ||
|
|
||
| // The deserialized chat history contains a list of function calls and the final answer to the question regarding the color of the sky in Boston. | ||
| chatHistory.AddUserMessage("Send the exact answer to my email: [email protected]"); | ||
| chatHistory.AddUserMessage("Send the exact answer to my email."); | ||
|
|
||
| var settings = new AzureOpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; | ||
|
|
||
|
|
@@ -832,7 +863,7 @@ public async Task ItShouldSupportNewFunctionCallingModelSerializedIntoChatHistor | |
|
|
||
| // Assert | ||
| Assert.Equal("[email protected]", emailRecipient); | ||
| Assert.Contains("61\u00B0F", emailBody); | ||
| Assert.Contains("61", emailBody); | ||
| } | ||
|
|
||
| /// <summary> | ||
|
|
@@ -867,7 +898,7 @@ public async Task SubsetOfFunctionsCanBeUsedForFunctionCallingAsync() | |
| // Arrange | ||
| var kernel = this.CreateAndInitializeKernel(); | ||
|
|
||
| var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday, "GetDayOfWeek", "Retrieves the current day of the week."); | ||
| var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday.ToString(), "GetDayOfWeek", "Retrieves the current day of the week."); | ||
| kernel.ImportPluginFromFunctions("HelperFunctions", [function]); | ||
|
|
||
| var chatHistory = new ChatHistory(); | ||
|
|
@@ -891,7 +922,7 @@ public async Task RequiredFunctionShouldBeCalledAsync() | |
| // Arrange | ||
| var kernel = this.CreateAndInitializeKernel(); | ||
|
|
||
| var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday, "GetDayOfWeek", "Retrieves the current day of the week."); | ||
| var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday.ToString(), "GetDayOfWeek", "Retrieves the current day of the week."); | ||
| kernel.ImportPluginFromFunctions("HelperFunctions", [function]); | ||
|
|
||
| var chatHistory = new ChatHistory(); | ||
|
|
@@ -939,20 +970,6 @@ private Kernel CreateAndInitializeKernel(bool importHelperPlugin = false) | |
| _ => "31 and snowing", | ||
| }; | ||
| }, "Get_Weather_For_City", "Gets the current weather for the specified city"), | ||
| kernel.CreateFunctionFromMethod((WeatherParameters parameters) => | ||
| { | ||
| if (parameters.City.Name == "Dublin" && (parameters.City.Country == "Ireland" || parameters.City.Country == "IE")) | ||
| { | ||
| return Task.FromResult(42.8); // 42.8 Fahrenheit. | ||
| } | ||
|
|
||
| throw new NotSupportedException($"Weather in {parameters.City.Name} ({parameters.City.Country}) is not supported."); | ||
| }, "Get_Current_Temperature", "Get current temperature."), | ||
| kernel.CreateFunctionFromMethod((double temperatureInFahrenheit) => | ||
| { | ||
| double temperatureInCelsius = (temperatureInFahrenheit - 32) * 5 / 9; | ||
| return Task.FromResult(temperatureInCelsius); | ||
| }, "Convert_Temperature_From_Fahrenheit_To_Celsius", "Convert temperature from Fahrenheit to Celsius.") | ||
| ]); | ||
| } | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.