From 8d2761a0c4c2c95745ee202a146c2155bbe974b7 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 29 Nov 2023 20:56:23 -0600 Subject: [PATCH 1/3] experiment failed. --- .../Models/FunctionCallingResponse.cs | 21 +++++ .../Functions/Models/FunctionDef.cs | 3 + .../Providers/ChatCompletionProvider.cs | 88 +++++++++++++++++-- .../templates/next_step_prompt.liquid | 2 +- .../templates/response_with_function.liquid | 9 ++ 5 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallingResponse.cs create mode 100644 src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallingResponse.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallingResponse.cs new file mode 100644 index 00000000..d523cb9c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallingResponse.cs @@ -0,0 +1,21 @@ +using System.Text.Json; + +namespace BotSharp.Abstraction.Functions.Models; + +/// +/// This class defines the LLM response output if function call needed +/// +public class FunctionCallingResponse +{ + [JsonPropertyName("role")] + public string Role { get; set; } = AgentRole.Assistant; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("function_name")] + public string? FunctionName { get; set; } + + [JsonPropertyName("args")] + public JsonDocument? Args { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs index d89620a9..54ef0000 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs @@ -4,7 +4,10 @@ public class FunctionDef { public string Name { get; set; } public string Description { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Impact { get; set; } + public FunctionParametersDef Parameters { get; set; } = new FunctionParametersDef(); public override string ToString() diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index 9733f39b..569d5d42 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -1,9 +1,13 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Routing; using BotSharp.Plugin.GoogleAI.Settings; using LLMSharp.Google.Palm; using Microsoft.Extensions.Logging; +using System.Diagnostics.Metrics; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.GoogleAI.Providers; @@ -33,18 +37,42 @@ public class ChatCompletionProvider : IChatCompletion hook.BeforeGenerating(agent, conversations)).ToArray()); var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey); - var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI")) - .ToList(); - var agentService = _services.GetRequiredService(); - var instruction = agentService.RenderedInstruction(agent); - var response = client.ChatAsync(messages, instruction, null).Result; + var (prompt, messages) = PrepareOptions(agent, conversations); - var message = response.Candidates.First(); - var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) + RoleDialogModel msg; + + if (messages == null) { - CurrentAgentId = agent.Id - }; + // use text completion + var response = client.GenerateTextAsync(prompt, null).Result; + + var message = response.Candidates.First(); + + // check if returns function calling + var llmResponse = message.Output.JsonContent(); + + msg = new RoleDialogModel(llmResponse.Role, llmResponse.Content) + { + CurrentAgentId = agent.Id, + FunctionName = llmResponse.FunctionName, + FunctionArgs = JsonSerializer.Serialize(llmResponse.Args) + }; + } + else + { + var response = client.ChatAsync(messages, context: prompt, examples: null, options: null).Result; + + var message = response.Candidates.First(); + + // check if returns function calling + var llmResponse = message.Content.JsonContent(); + + msg = new RoleDialogModel(llmResponse.Role, llmResponse.Content ?? message.Content) + { + CurrentAgentId = agent.Id + }; + } // After chat completion hook Task.WaitAll(hooks.Select(hook => @@ -56,6 +84,48 @@ public class ChatCompletionProvider : IChatCompletion return msg; } + private (string, List) PrepareOptions(Agent agent, List conversations) + { + var prompt = ""; + + var agentService = _services.GetRequiredService(); + + if (!string.IsNullOrEmpty(agent.Instruction)) + { + prompt += agentService.RenderedInstruction(agent); + } + + var routing = _services.GetRequiredService(); + var router = routing.Router; + + if (agent.Functions != null && agent.Functions.Count > 0) + { + prompt += "\r\n\r\n[Functions] defined in JSON Schema:\r\n"; + prompt += JsonSerializer.Serialize(agent.Functions, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }); + + prompt += "\r\n\r\n[Conversations]\r\n"; + foreach (var dialog in conversations) + { + prompt += dialog.Role == AgentRole.Function ? + $"{dialog.Role}: {dialog.FunctionName} => {dialog.Content}\r\n" : + $"{dialog.Role}: {dialog.Content}\r\n"; + } + + prompt += "\r\n\r\n" + router.Templates.FirstOrDefault(x => x.Name == "response_with_function").Content; + + return (prompt, null); + } + + var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI")) + .ToList(); + + return (prompt, messages); + } + public Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { throw new NotImplementedException(); diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid index 12de3ffd..2f74b54b 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid @@ -1,4 +1,4 @@ What is the next step based on the CONVERSATION? -Response must be in appropriate JSON format. +Response must be in required JSON format without any other contents. Route to the Agent that last handled the conversation if necessary. If user wants to speak to customer service, use function human_intervention_needed. \ No newline at end of file diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid new file mode 100644 index 00000000..4d47fee3 --- /dev/null +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid @@ -0,0 +1,9 @@ +1. Read the [Functions] definition, you can utilize the function to retrieve data or execute actions. +2. Think step by step, check if specific function will provider data to help complete user request based on the [Conversation]. +3. If you need to call a function to decide how to response user, + response in format: {"role": "function", "reason":"why choose this function", "function_name": "", "args": {}}, + otherwise response in format: {"role": "assistant", "reason":"why response to user", "content":""}. +4. If the [Conversation] already contains the function execution result, don't need to call it again. +5. If user mentioned some specific requirment, don't ask this again. + +Make your decision for the next step, output your response in JSON: \ No newline at end of file From 56f99c2924ab4154f5a5ffa0893ac9362c195817 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 30 Nov 2023 08:17:01 -0600 Subject: [PATCH 2/3] update function prompt. --- .../templates/response_with_function.liquid | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid index 4d47fee3..b9e97172 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid @@ -2,8 +2,9 @@ 2. Think step by step, check if specific function will provider data to help complete user request based on the [Conversation]. 3. If you need to call a function to decide how to response user, response in format: {"role": "function", "reason":"why choose this function", "function_name": "", "args": {}}, - otherwise response in format: {"role": "assistant", "reason":"why response to user", "content":""}. + otherwise response in format: {"role": "assistant", "reason":"why response to user", "content":"next step question"}. 4. If the [Conversation] already contains the function execution result, don't need to call it again. -5. If user mentioned some specific requirment, don't ask this again. +5. If user mentioned some specific requirment, don't ask this question in your response. +6. Don't repeat the same question in your response. -Make your decision for the next step, output your response in JSON: \ No newline at end of file +Which function should be used for the next step based on latest user's response, output your response in JSON: \ No newline at end of file From 9d4487836c3da7100cf2e833cd0789b902080cd6 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 30 Nov 2023 21:59:05 -0600 Subject: [PATCH 3/3] Add Temperature --- .../Providers/ChatCompletionProvider.cs | 34 ++++++++++++------- .../templates/response_with_function.liquid | 9 +++-- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index 175f810e..9e98e9b4 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -1,14 +1,12 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing; using BotSharp.Plugin.GoogleAI.Settings; using LLMSharp.Google.Palm; using Microsoft.Extensions.Logging; -using System.Diagnostics.Metrics; -using static System.Net.Mime.MediaTypeNames; +using LLMSharp.Google.Palm.DiscussService; namespace BotSharp.Plugin.GoogleAI.Providers; @@ -39,19 +37,25 @@ public class ChatCompletionProvider : IChatCompletion var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey); - var (prompt, messages) = PrepareOptions(agent, conversations); + var (prompt, messages, hasFunctions) = PrepareOptions(agent, conversations); RoleDialogModel msg; - if (messages == null) + if (hasFunctions) { // use text completion - var response = client.GenerateTextAsync(prompt, null).Result; + // var response = client.GenerateTextAsync(prompt, null).Result; + var response = client.ChatAsync(new PalmChatCompletionRequest + { + Context = prompt, + Messages = messages, + Temperature = 0.1f + }).Result; var message = response.Candidates.First(); // check if returns function calling - var llmResponse = message.Output.JsonContent(); + var llmResponse = message.Content.JsonContent(); msg = new RoleDialogModel(llmResponse.Role, llmResponse.Content) { @@ -79,13 +83,14 @@ public class ChatCompletionProvider : IChatCompletion Task.WaitAll(hooks.Select(hook => hook.AfterGenerated(msg, new TokenStatsModel { + Prompt = prompt, Model = _model })).ToArray()); return msg; } - private (string, List) PrepareOptions(Agent agent, List conversations) + private (string, List, bool) PrepareOptions(Agent agent, List conversations) { var prompt = ""; @@ -99,6 +104,9 @@ public class ChatCompletionProvider : IChatCompletion var routing = _services.GetRequiredService(); var router = routing.Router; + var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI")) + .ToList(); + if (agent.Functions != null && agent.Functions.Count > 0) { prompt += "\r\n\r\n[Functions] defined in JSON Schema:\r\n"; @@ -118,13 +126,13 @@ public class ChatCompletionProvider : IChatCompletion prompt += "\r\n\r\n" + router.Templates.FirstOrDefault(x => x.Name == "response_with_function").Content; - return (prompt, null); + return (prompt, new List + { + new PalmChatMessage("Which function should be used for the next step based on latest user or function response, output your response in JSON:", AgentRole.User), + }, true); } - var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI")) - .ToList(); - - return (prompt, messages); + return (prompt, messages, false); } public Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid index b9e97172..d45771b3 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/response_with_function.liquid @@ -1,10 +1,9 @@ +[Output Requirements] 1. Read the [Functions] definition, you can utilize the function to retrieve data or execute actions. -2. Think step by step, check if specific function will provider data to help complete user request based on the [Conversation]. +2. Think step by step, check if specific function will provider data to help complete user request based on the conversation. 3. If you need to call a function to decide how to response user, response in format: {"role": "function", "reason":"why choose this function", "function_name": "", "args": {}}, otherwise response in format: {"role": "assistant", "reason":"why response to user", "content":"next step question"}. -4. If the [Conversation] already contains the function execution result, don't need to call it again. +4. If the conversation already contains the function execution result, don't need to call it again. 5. If user mentioned some specific requirment, don't ask this question in your response. -6. Don't repeat the same question in your response. - -Which function should be used for the next step based on latest user's response, output your response in JSON: \ No newline at end of file +6. Don't repeat the same question in your response. \ No newline at end of file