From 3d3359cb951c40a7b04584e764fe3a837cfd3e74 Mon Sep 17 00:00:00 2001
From: Jicheng Lu <103353@smsassist.com>
Date: Tue, 14 May 2024 13:34:10 -0500
Subject: [PATCH] filter by model id and multi-modal

---
 .../MLTasks/ILlmProviderService.cs                  |  2 +-
 .../Infrastructures/CompletionProvider.cs           | 13 +++++++++----
 .../Infrastructures/LlmProviderService.cs           |  4 ++--
 .../Controllers/InstructModeController.cs           |  5 +----
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
index 75fd60e6..120304d2 100644
--- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
@@ -6,6 +6,6 @@ public interface ILlmProviderService
 {
     LlmModelSetting GetSetting(string provider, string model);
     List<string> GetProviders();
-    LlmModelSetting GetProviderModel(string provider, string id);
+    LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false);
     List<LlmModelSetting> GetProviderModels(string provider);
 }
diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
index bc0266ed..ace1e664 100644
--- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
+++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
@@ -35,10 +35,13 @@ public class CompletionProvider
     public static IChatCompletion GetChatCompletion(IServiceProvider services, 
         string? provider = null, 
         string? model = null,
+        string? modelId = null,
+        bool multiModal = false,
         AgentLlmConfig? agentConfig = null)
     {
         var completions = services.GetServices<IChatCompletion>();
-        (provider, model) = GetProviderAndModel(services, provider: provider, model: model, agentConfig: agentConfig);
+        (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, 
+            multiModal: multiModal, agentConfig: agentConfig);
 
         var completer = completions.FirstOrDefault(x => x.Provider == provider);
         if (completer == null)
@@ -55,6 +58,8 @@ public class CompletionProvider
     private static (string, string) GetProviderAndModel(IServiceProvider services, 
         string? provider = null,
         string? model = null,
+        string? modelId = null,
+        bool multiModal = false,
         AgentLlmConfig? agentConfig = null)
     {
         var agentSetting = services.GetRequiredService<AgentSettings>();
@@ -73,11 +78,11 @@ public class CompletionProvider
             {
                 model = state.GetState("model", model ?? "gpt-35-turbo-4k");
             }
-            else if (state.ContainsState("model_id"))
+            else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId))
             {
-                var modelId = state.GetState("model_id");
+                var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
                 var llmProviderService = services.GetRequiredService<ILlmProviderService>();
-                model = llmProviderService.GetProviderModel(provider, modelId)?.Name;
+                model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal)?.Name;
             }
         }
 
diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
index eb92ac51..7d92a687 100644
--- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
+++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
@@ -44,10 +44,10 @@ public class LlmProviderService : ILlmProviderService
             ?.Models ?? new List<LlmModelSetting>();
     }
 
-    public LlmModelSetting GetProviderModel(string provider, string id)
+    public LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false)
     {
         var models = GetProviderModels(provider)
-            .Where(x => x.Id == id)
+            .Where(x => x.Id == id && x.MultiModal == multiModal)
             .ToList();
 
         var random = new Random();
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
index 9f00a3e0..96fe3728 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
@@ -80,13 +80,10 @@ public class InstructModeController : ControllerBase
     {
         var state = _services.GetRequiredService<IConversationStateService>();
         input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
-        state.SetState("provider", input.Provider, source: StateSource.External)
-            .SetState("model", input.Model, source: StateSource.External)
-            .SetState("model_id", input.ModelId, source: StateSource.External);
 
         try
         {
-            var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo");
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4-turbo", multiModal: true);
             var message = await completion.GetChatCompletions(new Agent()
             {
                 Id = Guid.Empty.ToString(),